++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/scheduled-backup.log' Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/scheduled-backup.log ++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: the server doesn't have a resource type "projects" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' ']' +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.26 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.14.4 ++ '[' v3 == v2 ']' + main + create_namespace scheduled-backup-6083 + local namespace=scheduled-backup-6083 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + awk '{print$1}' + cleanup_rbac + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + xargs kubectl delete ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.nksvPS7Ciz + local LAST_OUT=/tmp/tmp.G0W4dWMn0A ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.YQBEpWjzvh + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + local LAST_ERR=/tmp/tmp.1e35lNx7HO + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl get ns + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.nksvPS7Ciz + cat /tmp/tmp.YQBEpWjzvh E0423 13:33:12.652218 23761 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:12.872654 23761 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:12.988781 23761 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:13.125340 23761 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request + rm /tmp/tmp.nksvPS7Ciz /tmp/tmp.YQBEpWjzvh + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level E0423 13:33:14.416345 23904 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:14.527856 23904 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:14.635466 23904 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:14.744798 23904 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request error: resource(s) were provided, but no name was specified + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.G0W4dWMn0A + cat /tmp/tmp.1e35lNx7HO E0423 13:33:20.473605 24237 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:20.785398 24237 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:20.892231 24237 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:20.998632 24237 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.G0W4dWMn0A /tmp/tmp.1e35lNx7HO + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + local LAST_OUT=/tmp/tmp.XZs9sSE1QO ++ mktemp + local LAST_ERR=/tmp/tmp.ZBWBseihH0 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.XZs9sSE1QO + cat /tmp/tmp.ZBWBseihH0 E0423 13:33:38.627470 25165 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:38.741972 25165 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:38.851316 25165 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:38.960543 25165 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.XZs9sSE1QO /tmp/tmp.ZBWBseihH0 + return 1 + true + kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth E0423 13:33:48.469656 25595 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:48.683098 25595 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:48.789944 25595 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0423 13:33:48.897132 25595 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found + true + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin delete namespace scheduled-backup-6083 ++ mktemp + local LAST_OUT=/tmp/tmp.JUeaFBsn1j ++ mktemp + local LAST_ERR=/tmp/tmp.qjqS1lhbT4 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete namespace scheduled-backup-6083 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete namespace scheduled-backup-6083 + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete namespace scheduled-backup-6083 + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.JUeaFBsn1j + cat /tmp/tmp.qjqS1lhbT4 Error from server (NotFound): namespaces "scheduled-backup-6083" not found + rm /tmp/tmp.JUeaFBsn1j /tmp/tmp.qjqS1lhbT4 + return 1 + : + wait_for_delete namespace/scheduled-backup-6083 + local res=namespace/scheduled-backup-6083 + set +o xtrace namespace/scheduled-backup-6083 - Error from server (NotFound): namespaces "scheduled-backup-6083" not found + kubectl_bin create namespace scheduled-backup-6083 ++ mktemp + local LAST_OUT=/tmp/tmp.hWWHpf2Dyz ++ mktemp + local LAST_ERR=/tmp/tmp.hh9DBZmcHk + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl create namespace scheduled-backup-6083 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.hWWHpf2Dyz namespace/scheduled-backup-6083 created + cat /tmp/tmp.hh9DBZmcHk + rm /tmp/tmp.hWWHpf2Dyz /tmp/tmp.hh9DBZmcHk + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.xM5F3W4RgU +++ mktemp ++ local LAST_ERR=/tmp/tmp.XC5WvuX2mc ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.xM5F3W4RgU ++ cat /tmp/tmp.XC5WvuX2mc ++ rm /tmp/tmp.xM5F3W4RgU /tmp/tmp.XC5WvuX2mc ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-scheduled-backup --namespace=scheduled-backup-6083 ++ mktemp + local LAST_OUT=/tmp/tmp.u8EOMu3q7R ++ mktemp + local LAST_ERR=/tmp/tmp.Yg7uxpb6yy + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-scheduled-backup --namespace=scheduled-backup-6083 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.u8EOMu3q7R Context "gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-scheduled-backup" modified. + cat /tmp/tmp.Yg7uxpb6yy + rm /tmp/tmp.u8EOMu3q7R /tmp/tmp.Yg7uxpb6yy + return 0 + deploy_operator + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml + local namespace_mode=disabled + local action=install + local namespace=scheduled-backup-6083 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml disabled install scheduled-backup-6083 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml + local namespace_mode=disabled + local operator_action=install + local namespace=scheduled-backup-6083 + local dont_send_telemetry=false + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml 'data[values.yaml]' + /usr/bin/sed -e 's#^namespace: .*#namespace: "scheduled-backup-6083"#g' + cat /dev/fd/63 - + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "scheduled-backup-6083"#g' ++ cut -d: -f1 ++ cut -d: -f2 ++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-721-a67b3475b"#g' ++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b ++ echo 'crunchy_debug: "true"' + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g' + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml metadata.namespace scheduled-backup-6083 + yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-deployer + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install + yq w -d3 - 'subjects[0].namespace' scheduled-backup-6083 ++ cat /tmp/tmp.yIkqhaYiTU/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "scheduled-backup-6083" namespace_mode: "disabled" pgo_image_prefix: "perconalab/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "PR-721-a67b3475b" pgo_operator_namespace: "scheduled-backup-6083"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.yIkqhaYiTU/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fg6xC8C0eM ++ mktemp + local LAST_ERR=/tmp/tmp.H8QG0Ampx8 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f /tmp/tmp.yIkqhaYiTU/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.fg6xC8C0eM serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.H8QG0Ampx8 + rm /tmp/tmp.fg6xC8C0eM /tmp/tmp.H8QG0Ampx8 + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.yIkqhaYiTU/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.jiwY5eLUJS ++ mktemp + local LAST_ERR=/tmp/tmp.DB1VOzjoUQ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete -f /tmp/tmp.yIkqhaYiTU/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.jiwY5eLUJS serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.DB1VOzjoUQ + rm /tmp/tmp.jiwY5eLUJS /tmp/tmp.DB1VOzjoUQ + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NXwDxEcEmc +++ mktemp ++ local LAST_ERR=/tmp/tmp.2nfRHSUtwx ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.NXwDxEcEmc ++ cat /tmp/tmp.2nfRHSUtwx ++ rm /tmp/tmp.NXwDxEcEmc /tmp/tmp.2nfRHSUtwx ++ return 0 + wait_pod postgres-operator-78b79fcc87-bjpf4 + local pod=postgres-operator-78b79fcc87-bjpf4 + local ns= + set +o xtrace postgres-operator-78b79fcc87-bjpf4true + [[ -n '' ]] + cluster=some-name-5931 + desc 'Testing full backup/restore from s3' + set +o xtrace ----------------------------------------------------------------------------------- Testing full backup/restore from s3 ----------------------------------------------------------------------------------- + apply_configs some-name-5931 + local cluster=some-name-5931 + set +x secret/some-name-5931-backrest-repo-config created + yq r /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/conf/custom-config.yml + kubectl_bin apply -f - + yq w - metadata.name some-name-5931-custom-config ++ mktemp + local LAST_OUT=/tmp/tmp.5tPmN9Mtmv ++ mktemp + local LAST_ERR=/tmp/tmp.eqfwwSQXdB + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5tPmN9Mtmv configmap/some-name-5931-custom-config created + cat /tmp/tmp.eqfwwSQXdB + rm /tmp/tmp.5tPmN9Mtmv /tmp/tmp.eqfwwSQXdB + return 0 + spinup_pgcluster some-name-5931 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml s3 some-name-5931-custom-config false true full + local cluster=some-name-5931 + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local backup=s3 + local custom_config=some-name-5931-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=full + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + local namespace=scheduled-backup-6083 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name-5931 scheduled-backup-6083 + local cluster=some-name-5931 + local namespace=scheduled-backup-6083 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + yq w - metadata.name some-name-5931-users-secret + kubectl_bin -n scheduled-backup-6083 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.eIErJuXQP0 + yq w - stringData.some-name some-name_pass ++ mktemp + local LAST_ERR=/tmp/tmp.jBKCDsfbdg + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.eIErJuXQP0 secret/some-name-5931-users-secret created + cat /tmp/tmp.jBKCDsfbdg + rm /tmp/tmp.eIErJuXQP0 /tmp/tmp.jBKCDsfbdg + return 0 + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + kubectl_bin -n scheduled-backup-6083 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.VJgtBPh9FT + yq w - metadata.name some-name-5931-pmm-secret ++ mktemp + local LAST_ERR=/tmp/tmp.zC2nQ4EOif + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.VJgtBPh9FT secret/some-name-5931-pmm-secret created + cat /tmp/tmp.zC2nQ4EOif + rm /tmp/tmp.VJgtBPh9FT /tmp/tmp.zC2nQ4EOif + return 0 + kubectl_bin -n scheduled-backup-6083 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.yVFsd6Gtkk ++ mktemp + local LAST_ERR=/tmp/tmp.V1uGNqrDTU + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.yVFsd6Gtkk deployment.apps/pg-client created + cat /tmp/tmp.V1uGNqrDTU + rm /tmp/tmp.yVFsd6Gtkk /tmp/tmp.V1uGNqrDTU + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name-5931 s3 false false some-name-5931-custom-config false true full false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local name=some-name-5931 + local backup=s3 + local restore_from=false + local restore_opts=false + local custom_config=some-name-5931-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=full + local tls_only=false + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name-5931 + yq w - metadata.labels.name some-name-5931 + yq w - metadata.labels.pg-cluster some-name-5931 + yq w - metadata.name some-name-5931 + yq w - spec.clustername some-name-5931 + yq w - spec.database some-name-5931 + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.user some-name + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.secretsName some-name-5931-users-secret + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.keepBackups false + yq w - spec.keepData false + yq w - spec.pmm.image perconalab/pmm-client:2.41.0 + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.name some-name-5931 + yq w - metadata.labels.deployment-name some-name-5931 + yq w - metadata.labels.crunchy-pgha-scope some-name-5931 + case ${podAntiAffinity} in + case ${backup} in + yq w /tmp/tmp.yIkqhaYiTU/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.storages[my-s3].type' s3 + [[ true == \t\r\u\e ]] + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.schedule[0].schedule' '*/5 * * * *' + yq w - 'spec.backup.schedule[0].type' full + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w - 'spec.backup.schedule[0].name' sch-backup + yq w - 'spec.backup.schedule[0].storage' my-s3 + yq w /tmp/tmp.yIkqhaYiTU/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].uriStyle' path + mv /tmp/tmp.yIkqhaYiTU/cr.backup.yaml /tmp/tmp.yIkqhaYiTU/cr.yaml + [[ false != \f\a\l\s\e ]] + [[ some-name-5931-custom-config != \f\a\l\s\e ]] + yq w -i /tmp/tmp.yIkqhaYiTU/cr.yaml spec.pgPrimary.customconfig some-name-5931-custom-config + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.yIkqhaYiTU/cr.yaml -n scheduled-backup-6083 ++ mktemp + local LAST_OUT=/tmp/tmp.truAirZAuw ++ mktemp + local LAST_ERR=/tmp/tmp.LSi5gZGCwz + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f /tmp/tmp.yIkqhaYiTU/cr.yaml -n scheduled-backup-6083 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.truAirZAuw perconapgcluster.pg.percona.com/some-name-5931 created + cat /tmp/tmp.LSi5gZGCwz + rm /tmp/tmp.truAirZAuw /tmp/tmp.LSi5gZGCwz + return 0 + wait_deployment some-name-5931-backrest-shared-repo + local name=some-name-5931-backrest-shared-repo + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-5931-backrest-shared-repo..... + wait_deployment some-name-5931 + local name=some-name-5931 + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-5931........... + [[ s3 == \g\c\s\+* ]] + wait_job_completion some-name-5931-stanza-create false + local job=some-name-5931-stanza-create + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' some-name-5931-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_job_completion backrest-backup-some-name-5931 + local job=backrest-backup-some-name-5931 + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-5931+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name-5931 + cluster_name=some-name-5931 + wait_cluster_status some-name-5931 'pgcluster Initialized' + local cluster_name=some-name-5931 + local 'target_status=pgcluster Initialized' + local target_namespace=scheduled-backup-6083 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1tI308LcwP +++ mktemp ++ local LAST_ERR=/tmp/tmp.dZmgwarDDR ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.1tI308LcwP ++ cat /tmp/tmp.dZmgwarDDR ++ rm /tmp/tmp.1tI308LcwP /tmp/tmp.dZmgwarDDR ++ return 0 + status_query='kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uvaul1rDcH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xmZc0VBqTo +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uvaul1rDcH +++ cat /tmp/tmp.xmZc0VBqTo +++ rm /tmp/tmp.uvaul1rDcH /tmp/tmp.xmZc0VBqTo +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name-5931 ++ local user=postgres ++ local cluster=some-name-5931 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.u03rFUhz0r ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YNkWm4nCTN +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.u03rFUhz0r +++ cat /tmp/tmp.YNkWm4nCTN +++ rm /tmp/tmp.u03rFUhz0r /tmp/tmp.YNkWm4nCTN +++ return 0 ++ [[ xsome-name-5931 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.87SChB5FBh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GqX3mHX0R7 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.87SChB5FBh +++ cat /tmp/tmp.GqX3mHX0R7 +++ rm /tmp/tmp.87SChB5FBh /tmp/tmp.GqX3mHX0R7 +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2qv5A0DmJw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iQvyOzHQBK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2qv5A0DmJw +++ cat /tmp/tmp.iQvyOzHQBK +++ rm /tmp/tmp.2qv5A0DmJw /tmp/tmp.iQvyOzHQBK +++ return 0 ++ [[ xsome-name-5931-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.JnpKuCf0IY +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.082xVyzu04 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.JnpKuCf0IY ++++ cat /tmp/tmp.082xVyzu04 ++++ rm /tmp/tmp.JnpKuCf0IY /tmp/tmp.082xVyzu04 ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zgS2N1nlUY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.g6auE61OrP +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zgS2N1nlUY +++ cat /tmp/tmp.g6auE61OrP +++ rm /tmp/tmp.zgS2N1nlUY /tmp/tmp.g6auE61OrP +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nn2nMpssyx +++ mktemp ++ local LAST_ERR=/tmp/tmp.UWvdvDgrDh ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.nn2nMpssyx ++ cat /tmp/tmp.UWvdvDgrDh ++ rm /tmp/tmp.nn2nMpssyx /tmp/tmp.UWvdvDgrDh ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.p0yXD4Tk6C ++ mktemp + local LAST_ERR=/tmp/tmp.LkXC1yh1cw + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.p0yXD4Tk6C + cat /tmp/tmp.LkXC1yh1cw + rm /tmp/tmp.p0yXD4Tk6C /tmp/tmp.LkXC1yh1cw + return 0 ++ get_psql_user_pass postgres some-name-5931 ++ local user=postgres ++ local cluster=some-name-5931 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QcdlXfy02V ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9nSgiRlRev +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.QcdlXfy02V +++ cat /tmp/tmp.9nSgiRlRev +++ rm /tmp/tmp.QcdlXfy02V /tmp/tmp.9nSgiRlRev +++ return 0 ++ [[ xsome-name-5931 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uPxqh3g7ba ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hMYWlFLL70 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uPxqh3g7ba +++ cat /tmp/tmp.hMYWlFLL70 +++ rm /tmp/tmp.uPxqh3g7ba /tmp/tmp.hMYWlFLL70 +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NmPF8Qubik ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6A3kMDMy2O +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.NmPF8Qubik +++ cat /tmp/tmp.6A3kMDMy2O +++ rm /tmp/tmp.NmPF8Qubik /tmp/tmp.6A3kMDMy2O +++ return 0 ++ [[ xsome-name-5931-users-secret == \x ]] ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ base64 -d +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.fFa0pVVHL5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PcgtwD8BZ0 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.fFa0pVVHL5 ++++ cat /tmp/tmp.PcgtwD8BZ0 ++++ rm /tmp/tmp.fFa0pVVHL5 /tmp/tmp.PcgtwD8BZ0 ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zcbfmIn3OY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LR27QGqPM1 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zcbfmIn3OY +++ cat /tmp/tmp.LR27QGqPM1 +++ rm /tmp/tmp.zcbfmIn3OY /tmp/tmp.LR27QGqPM1 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0gXYGxAurB +++ mktemp ++ local LAST_ERR=/tmp/tmp.kbfeYyKWkY ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.0gXYGxAurB ++ cat /tmp/tmp.kbfeYyKWkY ++ rm /tmp/tmp.0gXYGxAurB /tmp/tmp.kbfeYyKWkY ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.8XQmBW4zD9 ++ mktemp + local LAST_ERR=/tmp/tmp.NcmVgdPfip + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8XQmBW4zD9 + cat /tmp/tmp.NcmVgdPfip + rm /tmp/tmp.8XQmBW4zD9 /tmp/tmp.NcmVgdPfip + return 0 ++ get_psql_user_pass postgres some-name-5931 ++ local user=postgres ++ local cluster=some-name-5931 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GEAQS4j5yh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YeNKgTFOj1 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.GEAQS4j5yh +++ cat /tmp/tmp.YeNKgTFOj1 +++ rm /tmp/tmp.GEAQS4j5yh /tmp/tmp.YeNKgTFOj1 +++ return 0 ++ [[ xsome-name-5931 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.unZxKgwF4Q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.31xAFJ7c6T +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.unZxKgwF4Q +++ cat /tmp/tmp.31xAFJ7c6T +++ rm /tmp/tmp.unZxKgwF4Q /tmp/tmp.31xAFJ7c6T +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sXUUlflAeI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.g7xmzBtBfo +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.sXUUlflAeI +++ cat /tmp/tmp.g7xmzBtBfo +++ rm /tmp/tmp.sXUUlflAeI /tmp/tmp.g7xmzBtBfo +++ return 0 ++ [[ xsome-name-5931-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.SjwkYaBnJG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.c72mAndKCF ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.SjwkYaBnJG ++++ cat /tmp/tmp.c72mAndKCF ++++ rm /tmp/tmp.SjwkYaBnJG /tmp/tmp.c72mAndKCF ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UvF5ISFj23 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5ymiOLhTfP +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.UvF5ISFj23 +++ cat /tmp/tmp.5ymiOLhTfP +++ rm /tmp/tmp.UvF5ISFj23 /tmp/tmp.5ymiOLhTfP +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2jCmkq8mva +++ mktemp ++ local LAST_ERR=/tmp/tmp.BsuxsEsYtV ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.2jCmkq8mva ++ cat /tmp/tmp.BsuxsEsYtV ++ rm /tmp/tmp.2jCmkq8mva /tmp/tmp.BsuxsEsYtV ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.cgGTkDIbZL ++ mktemp + local LAST_ERR=/tmp/tmp.qztctq82kg + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.cgGTkDIbZL + cat /tmp/tmp.qztctq82kg + rm /tmp/tmp.cgGTkDIbZL /tmp/tmp.qztctq82kg + return 0 + sleep 10 +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.backup.schedule[0].name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.428cUrT7Yr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6h4DSpDNZ4 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.backup.schedule[0].name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.428cUrT7Yr +++ cat /tmp/tmp.6h4DSpDNZ4 +++ rm /tmp/tmp.428cUrT7Yr /tmp/tmp.6h4DSpDNZ4 +++ return 0 ++ next_backup_after_sec some-name-5931-sch-backup ++ local conf_map_name=some-name-5931-sch-backup ++++ kubectl_bin get configmap/some-name-5931-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ra9rSAzOSD +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.l8LyMCaz9e ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl get configmap/some-name-5931-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ra9rSAzOSD ++++ cat /tmp/tmp.l8LyMCaz9e ++++ rm /tmp/tmp.ra9rSAzOSD /tmp/tmp.l8LyMCaz9e ++++ return 0 +++ /usr/bin/date +%s --date=2024-04-23T13:36:20Z --utc ++ local configmap_creation_time=1713879380 +++ kubectl_bin get configmap/some-name-5931-sch-backup -o 'jsonpath={.data.schedule}' +++ jq .schedule +++ grep -Eo '[0-9]+' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VWtTS0RF7o ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0DW9ldkJxt +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get configmap/some-name-5931-sch-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.VWtTS0RF7o +++ cat /tmp/tmp.0DW9ldkJxt +++ rm /tmp/tmp.VWtTS0RF7o /tmp/tmp.0DW9ldkJxt +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1713879380 300 +++ local start_time_sec=1713879380 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1713879680 +++ local next_interval_min_edge=80 +++ local next_backup_at_sec=1713879600 +++ echo 1713879600 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Tue, 23 Apr 2024 13:40:40 GMT ' ++++ /usr/bin/date --date 'Tue, 23 Apr 2024 13:40:40 GMT ' --utc +%s +++ echo 1713879640 ++ local result=-40 ++ [[ -40 -gt 0 ]] ++++ get_current_epoch_time +++++ curl -sI https://percona.com/ +++++ grep date +++++ /usr/bin/sed 's/date: //g' ++++ local 'curent_date=Tue, 23 Apr 2024 13:40:41 GMT ' +++++ /usr/bin/date --date 'Tue, 23 Apr 2024 13:40:41 GMT ' --utc +%s ++++ echo 1713879641 +++ calculate_next_backup_start 1713879641 300 +++ local start_time_sec=1713879641 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1713879941 +++ local next_interval_min_edge=41 +++ local next_backup_at_sec=1713879900 +++ echo 1713879900 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Tue, 23 Apr 2024 13:40:41 GMT ' ++++ /usr/bin/date --date 'Tue, 23 Apr 2024 13:40:41 GMT ' --utc +%s +++ echo 1713879641 ++ result=259 ++ [[ 259 -gt 0 ]] ++ echo 259 + sleep 259 + kubectl_bin wait job/some-name-5931-full-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.uSF1nMYPVA ++ mktemp + local LAST_ERR=/tmp/tmp.sznXuTrH81 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl wait job/some-name-5931-full-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.uSF1nMYPVA job.batch/some-name-5931-full-sch-backup condition met + cat /tmp/tmp.sznXuTrH81 + rm /tmp/tmp.uSF1nMYPVA /tmp/tmp.sznXuTrH81 + return 0 + restore some-name-5931 s3 full + local cluster=some-name-5931 + local storage=s3 + local backup_type=full + local restore_target= ++ get_psql_user_pass postgres some-name-5931 ++ local user=postgres ++ local cluster=some-name-5931 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZTr7rlQdvJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vzYwttJnKD +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ZTr7rlQdvJ +++ cat /tmp/tmp.vzYwttJnKD +++ rm /tmp/tmp.ZTr7rlQdvJ /tmp/tmp.vzYwttJnKD +++ return 0 ++ [[ xsome-name-5931 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.evdp80xYAz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Avv4fPazXF +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.evdp80xYAz +++ cat /tmp/tmp.Avv4fPazXF +++ rm /tmp/tmp.evdp80xYAz /tmp/tmp.Avv4fPazXF +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ym517ZFnw3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SKYukndCpP +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ym517ZFnw3 +++ cat /tmp/tmp.SKYukndCpP +++ rm /tmp/tmp.ym517ZFnw3 /tmp/tmp.SKYukndCpP +++ return 0 ++ [[ xsome-name-5931-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.A2bDNBEO5j +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.0A5wlh1Zfl ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.A2bDNBEO5j ++++ cat /tmp/tmp.0A5wlh1Zfl ++++ rm /tmp/tmp.A2bDNBEO5j /tmp/tmp.0A5wlh1Zfl ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CN0uuvaS1Z ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nWezwn7QFH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.CN0uuvaS1Z +++ cat /tmp/tmp.nWezwn7QFH +++ rm /tmp/tmp.CN0uuvaS1Z /tmp/tmp.nWezwn7QFH +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9oOJuNsyfJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.BywFKkQr8A ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.9oOJuNsyfJ ++ cat /tmp/tmp.BywFKkQr8A ++ rm /tmp/tmp.9oOJuNsyfJ /tmp/tmp.BywFKkQr8A ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.PoWJAm5uq2 ++ mktemp + local LAST_ERR=/tmp/tmp.4TRIZlirTS + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.PoWJAm5uq2 + cat /tmp/tmp.4TRIZlirTS + rm /tmp/tmp.PoWJAm5uq2 /tmp/tmp.4TRIZlirTS + return 0 ++ get_psql_user_pass postgres some-name-5931 ++ local user=postgres ++ local cluster=some-name-5931 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9FY8Pzsj90 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DpXqGv8bhY +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.9FY8Pzsj90 +++ cat /tmp/tmp.DpXqGv8bhY +++ rm /tmp/tmp.9FY8Pzsj90 /tmp/tmp.DpXqGv8bhY +++ return 0 ++ [[ xsome-name-5931 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.X9mvsmSMUS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XdjqBTgVwL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.X9mvsmSMUS +++ cat /tmp/tmp.XdjqBTgVwL +++ rm /tmp/tmp.X9mvsmSMUS /tmp/tmp.XdjqBTgVwL +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.48cZlsHZC5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6uVlktEk2i +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.48cZlsHZC5 +++ cat /tmp/tmp.6uVlktEk2i +++ rm /tmp/tmp.48cZlsHZC5 /tmp/tmp.6uVlktEk2i +++ return 0 ++ [[ xsome-name-5931-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Ui08EX6lcQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.8LKAxW4GrL ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.Ui08EX6lcQ ++++ cat /tmp/tmp.8LKAxW4GrL ++++ rm /tmp/tmp.Ui08EX6lcQ /tmp/tmp.8LKAxW4GrL ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0uJ5Zj0GZl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.L7s6B8ZzyH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.0uJ5Zj0GZl +++ cat /tmp/tmp.L7s6B8ZzyH +++ rm /tmp/tmp.0uJ5Zj0GZl /tmp/tmp.L7s6B8ZzyH +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-5931.scheduled-backup-6083 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.onwmNjTABi +++ mktemp ++ local LAST_ERR=/tmp/tmp.YcALj7OgSy ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.onwmNjTABi ++ cat /tmp/tmp.YcALj7OgSy ++ rm /tmp/tmp.onwmNjTABi /tmp/tmp.YcALj7OgSy ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.8FI3P00XvG ++ mktemp + local LAST_ERR=/tmp/tmp.GX5mzw6r70 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8FI3P00XvG + cat /tmp/tmp.GX5mzw6r70 + rm /tmp/tmp.8FI3P00XvG /tmp/tmp.GX5mzw6r70 + return 0 + '[' '!' -s /tmp/tmp.yIkqhaYiTU/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.yIkqhaYiTU/select-1.sql ++ get_psql_user_pass postgres some-name-5931 ++ local user=postgres ++ local cluster=some-name-5931 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oijTrF86nr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BJlQ7xdaOR +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.oijTrF86nr +++ cat /tmp/tmp.BJlQ7xdaOR +++ rm /tmp/tmp.oijTrF86nr /tmp/tmp.BJlQ7xdaOR +++ return 0 ++ [[ xsome-name-5931 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fyMCP6uzzb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.QhbRZt9Aaq +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.fyMCP6uzzb +++ cat /tmp/tmp.QhbRZt9Aaq +++ rm /tmp/tmp.fyMCP6uzzb /tmp/tmp.QhbRZt9Aaq +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aTbcG2AOye ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3wjUSaf2Vi +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.aTbcG2AOye +++ cat /tmp/tmp.3wjUSaf2Vi +++ rm /tmp/tmp.aTbcG2AOye /tmp/tmp.3wjUSaf2Vi +++ return 0 ++ [[ xsome-name-5931-users-secret == \x ]] ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ mktemp +++ base64 -d ++++ local LAST_OUT=/tmp/tmp.nU10f3Ev1g +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.1xhtyrQ10u ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.nU10f3Ev1g ++++ cat /tmp/tmp.1xhtyrQ10u ++++ rm /tmp/tmp.nU10f3Ev1g /tmp/tmp.1xhtyrQ10u ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mtle0ILOWZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kQ0rt2olfT +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.mtle0ILOWZ +++ cat /tmp/tmp.kQ0rt2olfT +++ rm /tmp/tmp.mtle0ILOWZ /tmp/tmp.kQ0rt2olfT +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BblcAInpPm +++ mktemp ++ local LAST_ERR=/tmp/tmp.O5X81GB3ap ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.BblcAInpPm ++ cat /tmp/tmp.O5X81GB3ap ++ rm /tmp/tmp.BblcAInpPm /tmp/tmp.O5X81GB3ap ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.Yq5ZLPiYkS ++ mktemp + local LAST_ERR=/tmp/tmp.0sArl030hI + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Yq5ZLPiYkS + cat /tmp/tmp.0sArl030hI + rm /tmp/tmp.Yq5ZLPiYkS /tmp/tmp.0sArl030hI + return 0 ++ awk '{$1=$1;print}' +++ get_psql_user_pass postgres some-name-5931 +++ local user=postgres +++ local cluster=some-name-5931 +++ local namespace=scheduled-backup-6083 ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.jDnYiOL3ql +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2MmcZz9IMT ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.jDnYiOL3ql ++++ cat /tmp/tmp.2MmcZz9IMT ++++ rm /tmp/tmp.jDnYiOL3ql /tmp/tmp.2MmcZz9IMT ++++ return 0 +++ [[ xsome-name-5931 == \x ]] +++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.NkCqs97h7o +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.gP8izIlyIU ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.NkCqs97h7o ++++ cat /tmp/tmp.gP8izIlyIU ++++ rm /tmp/tmp.NkCqs97h7o /tmp/tmp.gP8izIlyIU ++++ return 0 ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.n8CeJUfpzH +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.n1IZ88Jjoh ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.n8CeJUfpzH ++++ cat /tmp/tmp.n1IZ88Jjoh ++++ rm /tmp/tmp.n8CeJUfpzH /tmp/tmp.n1IZ88Jjoh ++++ return 0 +++ [[ xsome-name-5931-users-secret == \x ]] ++++ base64 -d +++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.qRBFxf3I22 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.2RU9b43St7 +++++ local exit_status=0 +++++ local timeout=4 +++++ set +o errexit ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ set -o errexit +++++ cat /tmp/tmp.qRBFxf3I22 +++++ cat /tmp/tmp.2RU9b43St7 +++++ rm /tmp/tmp.qRBFxf3I22 /tmp/tmp.2RU9b43St7 +++++ return 0 ++++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.IVV6mUEXzr +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.VLm8CaNtxi ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.IVV6mUEXzr ++++ cat /tmp/tmp.VLm8CaNtxi ++++ rm /tmp/tmp.IVV6mUEXzr /tmp/tmp.VLm8CaNtxi ++++ return 0 +++ echo postgres_password ++ run_psql '\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' postgres:postgres_password@some-name-5931.scheduled-backup-6083 ++ local 'command=\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' ++ local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 ++ local driver=postgres ++ local suffix=.svc.cluster.local +++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.agmusZ7oUT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.stQGFUCADa +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.agmusZ7oUT +++ cat /tmp/tmp.stQGFUCADa +++ rm /tmp/tmp.agmusZ7oUT /tmp/tmp.stQGFUCADa +++ return 0 ++ local client_container=pg-client-76df9df876-4grks ++ kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jvgMmBknU7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mA66UWPs1J ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.jvgMmBknU7 ++ cat /tmp/tmp.mA66UWPs1J ++ rm /tmp/tmp.jvgMmBknU7 /tmp/tmp.mA66UWPs1J ++ return 0 + restore_target=773 ++ get_psql_user_pass postgres some-name-5931 ++ local user=postgres ++ local cluster=some-name-5931 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gTLpcsucmL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5IzzDzAd9X +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.gTLpcsucmL +++ cat /tmp/tmp.5IzzDzAd9X +++ rm /tmp/tmp.gTLpcsucmL /tmp/tmp.5IzzDzAd9X +++ return 0 ++ [[ xsome-name-5931 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kFt6C1iMjS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eEeEZ8zv7K +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.kFt6C1iMjS +++ cat /tmp/tmp.eEeEZ8zv7K +++ rm /tmp/tmp.kFt6C1iMjS /tmp/tmp.eEeEZ8zv7K +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mUKeszV2eS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qi7nYFEwu0 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.mUKeszV2eS +++ cat /tmp/tmp.qi7nYFEwu0 +++ rm /tmp/tmp.mUKeszV2eS /tmp/tmp.qi7nYFEwu0 +++ return 0 ++ [[ xsome-name-5931-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.y5PrbD3B5u +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.3zMvtf6m0w ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.y5PrbD3B5u ++++ cat /tmp/tmp.3zMvtf6m0w ++++ rm /tmp/tmp.y5PrbD3B5u /tmp/tmp.3zMvtf6m0w ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ph3P50LzGf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lPI3mGCxEO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ph3P50LzGf +++ cat /tmp/tmp.lPI3mGCxEO +++ rm /tmp/tmp.ph3P50LzGf /tmp/tmp.lPI3mGCxEO +++ return 0 ++ echo postgres_password + run_psql 'DROP DATABASE myapp' postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local 'command=DROP DATABASE myapp' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gS0YboeBtN +++ mktemp ++ local LAST_ERR=/tmp/tmp.80nmv9uue4 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.gS0YboeBtN ++ cat /tmp/tmp.80nmv9uue4 ++ rm /tmp/tmp.gS0YboeBtN /tmp/tmp.80nmv9uue4 ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.h5cqrkAsqP ++ mktemp + local LAST_ERR=/tmp/tmp.Z86epUKyfH + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.h5cqrkAsqP + cat /tmp/tmp.Z86epUKyfH + rm /tmp/tmp.h5cqrkAsqP /tmp/tmp.Z86epUKyfH + return 0 + run_restore some-name-5931 full s3 773 xid + local cluster=some-name-5931 + local rstr_prefix=full + local storage=s3 + local target=773 + local type=xid + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/conf/restore.yml metadata.labels.pg-cluster some-name-5931 + yq w - metadata.name full-some-name-5931 + yq w - spec.name full-some-name-5931 + yq w - spec.parameters.backrest-storage-type s3 + yq w - spec.namespace scheduled-backup-6083 + yq w - spec.parameters.backrest-restore-cluster some-name-5931 + [[ 773 != \n\u\l\l ]] + yq w /tmp/tmp.yIkqhaYiTU/restore.yml --style=single spec.parameters.backrest-pitr-target 773 + yq w --style=single -- - spec.parameters.backrest-restore-opts --type=xid + mv /tmp/tmp.yIkqhaYiTU/restore.pitr.yml /tmp/tmp.yIkqhaYiTU/restore.yml + kubectl apply -f /tmp/tmp.yIkqhaYiTU/restore.yml pgtask.pg.percona.com/full-some-name-5931 created + wait_bootstrap_completeness some-name-5931 + local cluster=some-name-5931 + wait_job_completion some-name-5931-bootstrap + local job=some-name-5931-bootstrap + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' some-name-5931-bootstrap+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-bootstrap -o 'jsonpath={.metadata.name}' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_deployment some-name-5931-backrest-shared-repo + local name=some-name-5931-backrest-shared-repo + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-5931-backrest-shared-repo + wait_deployment some-name-5931 + local name=some-name-5931 + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-5931 + wait_job_completion some-name-5931-stanza-create false + local job=some-name-5931-stanza-create + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' some-name-5931-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name-5931 + local job=backrest-backup-some-name-5931 + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-5931+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 48 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 49 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-5931 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo ++ get_psql_user_pass postgres some-name-5931 ++ local user=postgres ++ local cluster=some-name-5931 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.N8hM6KOHfs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.J6haTOtMho +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.N8hM6KOHfs +++ cat /tmp/tmp.J6haTOtMho +++ rm /tmp/tmp.N8hM6KOHfs /tmp/tmp.J6haTOtMho +++ return 0 ++ [[ xsome-name-5931 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qD7KDeKKOA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FjjrwNjyJL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-5931 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.qD7KDeKKOA +++ cat /tmp/tmp.FjjrwNjyJL +++ rm /tmp/tmp.qD7KDeKKOA /tmp/tmp.FjjrwNjyJL +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tyIo3PVKfy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qtqLELPRNK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.tyIo3PVKfy +++ cat /tmp/tmp.qtqLELPRNK +++ rm /tmp/tmp.tyIo3PVKfy /tmp/tmp.qtqLELPRNK +++ return 0 ++ [[ xsome-name-5931-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.XMI5SgvQKH +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.GXlKFGgaeS ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-5931 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.XMI5SgvQKH ++++ cat /tmp/tmp.GXlKFGgaeS ++++ rm /tmp/tmp.XMI5SgvQKH /tmp/tmp.GXlKFGgaeS ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6LN5kbfpj5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2P0LL7qtPf +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-5931-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.6LN5kbfpj5 +++ cat /tmp/tmp.2P0LL7qtPf +++ rm /tmp/tmp.6LN5kbfpj5 /tmp/tmp.2P0LL7qtPf +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-5931.scheduled-backup-6083 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-5931.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7xM1TU9hSZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.cz3GjzVTRI ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.7xM1TU9hSZ ++ cat /tmp/tmp.cz3GjzVTRI ++ rm /tmp/tmp.7xM1TU9hSZ /tmp/tmp.cz3GjzVTRI ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.TLIXdsPYE3 ++ mktemp + local LAST_ERR=/tmp/tmp.r4f6IA2Qwp + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-5931.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.TLIXdsPYE3 + cat /tmp/tmp.r4f6IA2Qwp + rm /tmp/tmp.TLIXdsPYE3 /tmp/tmp.r4f6IA2Qwp + return 0 + '[' '!' -s /tmp/tmp.yIkqhaYiTU/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.yIkqhaYiTU/select-1.sql + delete_cluster_gracefully some-name-5931 + local cluster=some-name-5931 + kubectl_bin delete perconapgcluster/some-name-5931 ++ mktemp + local LAST_OUT=/tmp/tmp.JQ4cMJX12t ++ mktemp + local LAST_ERR=/tmp/tmp.nePI6FICo7 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete perconapgcluster/some-name-5931 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.JQ4cMJX12t perconapgcluster.pg.percona.com "some-name-5931" deleted + cat /tmp/tmp.nePI6FICo7 + rm /tmp/tmp.JQ4cMJX12t /tmp/tmp.nePI6FICo7 + return 0 ++ kubectl_bin -n scheduled-backup-6083 get jobs --selector=pg-cluster=some-name-5931,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gG2GwcO04N +++ mktemp ++ local LAST_ERR=/tmp/tmp.amWnBQe42T ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get jobs --selector=pg-cluster=some-name-5931,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.gG2GwcO04N ++ cat /tmp/tmp.amWnBQe42T ++ rm /tmp/tmp.gG2GwcO04N /tmp/tmp.amWnBQe42T ++ return 0 + rmjob_name=some-name-5931-rmdata-autu + wait_job_completion some-name-5931-rmdata-autu false + local job=some-name-5931-rmdata-autu + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' some-name-5931-rmdata-autu+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-5931-rmdata-autu -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + desc 'Testing diff backup/restore from s3' + set +o xtrace ----------------------------------------------------------------------------------- Testing diff backup/restore from s3 ----------------------------------------------------------------------------------- + cluster=some-name-4738 + apply_configs some-name-4738 + local cluster=some-name-4738 + set +x secret/some-name-4738-backrest-repo-config created + kubectl_bin apply -f - + yq r /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/conf/custom-config.yml + yq w - metadata.name some-name-4738-custom-config ++ mktemp + local LAST_OUT=/tmp/tmp.YSaCn21Omm ++ mktemp + local LAST_ERR=/tmp/tmp.8f07kpLTzy + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.YSaCn21Omm configmap/some-name-4738-custom-config created + cat /tmp/tmp.8f07kpLTzy + rm /tmp/tmp.YSaCn21Omm /tmp/tmp.8f07kpLTzy + return 0 + spinup_pgcluster some-name-4738 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml s3 some-name-4738-custom-config false true diff + local cluster=some-name-4738 + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local backup=s3 + local custom_config=some-name-4738-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=diff + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + local namespace=scheduled-backup-6083 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name-4738 scheduled-backup-6083 + local cluster=some-name-4738 + local namespace=scheduled-backup-6083 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + yq w - metadata.name some-name-4738-users-secret + yq w - stringData.some-name some-name_pass + kubectl_bin -n scheduled-backup-6083 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.95OKtGwWHr ++ mktemp + local LAST_ERR=/tmp/tmp.HOkpwLPo5h + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.95OKtGwWHr secret/some-name-4738-users-secret created + cat /tmp/tmp.HOkpwLPo5h + rm /tmp/tmp.95OKtGwWHr /tmp/tmp.HOkpwLPo5h + return 0 + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + yq w - metadata.name some-name-4738-pmm-secret + kubectl_bin -n scheduled-backup-6083 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.9Ck0pGrVeC ++ mktemp + local LAST_ERR=/tmp/tmp.lubID5xoTu + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.9Ck0pGrVeC secret/some-name-4738-pmm-secret created + cat /tmp/tmp.lubID5xoTu + rm /tmp/tmp.9Ck0pGrVeC /tmp/tmp.lubID5xoTu + return 0 + kubectl_bin -n scheduled-backup-6083 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Ar0Yr77mWs ++ mktemp + local LAST_ERR=/tmp/tmp.36EcM3wGzT + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Ar0Yr77mWs deployment.apps/pg-client unchanged + cat /tmp/tmp.36EcM3wGzT + rm /tmp/tmp.Ar0Yr77mWs /tmp/tmp.36EcM3wGzT + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name-4738 s3 false false some-name-4738-custom-config false true diff false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local name=some-name-4738 + local backup=s3 + local restore_from=false + local restore_opts=false + local custom_config=some-name-4738-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=diff + local tls_only=false + local root_via_pgbouncer= + yq w - metadata.labels.crunchy-pgha-scope some-name-4738 + yq w - metadata.labels.deployment-name some-name-4738 + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name-4738 + yq w - metadata.labels.name some-name-4738 + yq w - metadata.name some-name-4738 + yq w - metadata.labels.pg-cluster some-name-4738 + yq w - spec.secretsName some-name-4738-users-secret + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.keepBackups false + yq w - spec.keepData false + yq w - spec.name some-name-4738 + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.user some-name + yq w - spec.database some-name-4738 + yq w - spec.pmm.image perconalab/pmm-client:2.41.0 + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.clustername some-name-4738 + case ${podAntiAffinity} in + case ${backup} in + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w /tmp/tmp.yIkqhaYiTU/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].uriStyle' path + [[ true == \t\r\u\e ]] + yq w /tmp/tmp.yIkqhaYiTU/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w - 'spec.backup.schedule[0].type' diff + yq w - 'spec.backup.schedule[0].schedule' '*/5 * * * *' + yq w - 'spec.backup.schedule[0].storage' my-s3 + yq w - 'spec.backup.schedule[0].name' sch-backup + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + mv /tmp/tmp.yIkqhaYiTU/cr.backup.yaml /tmp/tmp.yIkqhaYiTU/cr.yaml + [[ false != \f\a\l\s\e ]] + [[ some-name-4738-custom-config != \f\a\l\s\e ]] + yq w -i /tmp/tmp.yIkqhaYiTU/cr.yaml spec.pgPrimary.customconfig some-name-4738-custom-config + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.yIkqhaYiTU/cr.yaml -n scheduled-backup-6083 ++ mktemp + local LAST_OUT=/tmp/tmp.lXxPGrIAQo ++ mktemp + local LAST_ERR=/tmp/tmp.HyQbVXt1Xv + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f /tmp/tmp.yIkqhaYiTU/cr.yaml -n scheduled-backup-6083 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.lXxPGrIAQo perconapgcluster.pg.percona.com/some-name-4738 created + cat /tmp/tmp.HyQbVXt1Xv + rm /tmp/tmp.lXxPGrIAQo /tmp/tmp.HyQbVXt1Xv + return 0 + wait_deployment some-name-4738-backrest-shared-repo + local name=some-name-4738-backrest-shared-repo + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-4738-backrest-shared-repo.. + wait_deployment some-name-4738 + local name=some-name-4738 + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-4738..... + [[ s3 == \g\c\s\+* ]] + wait_job_completion some-name-4738-stanza-create false + local job=some-name-4738-stanza-create + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' some-name-4738-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_job_completion backrest-backup-some-name-4738 + local job=backrest-backup-some-name-4738 + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-4738+ retry=0 ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name-4738 + cluster_name=some-name-4738 + wait_cluster_status some-name-4738 'pgcluster Initialized' + local cluster_name=some-name-4738 + local 'target_status=pgcluster Initialized' + local target_namespace=scheduled-backup-6083 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wYzLM0lrWc +++ mktemp ++ local LAST_ERR=/tmp/tmp.eHL30gn1bD ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.wYzLM0lrWc ++ cat /tmp/tmp.eHL30gn1bD ++ rm /tmp/tmp.wYzLM0lrWc /tmp/tmp.eHL30gn1bD ++ return 0 + status_query='kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tRsYX5X5VV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.owSfrAJFdb +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.tRsYX5X5VV +++ cat /tmp/tmp.owSfrAJFdb +++ rm /tmp/tmp.tRsYX5X5VV /tmp/tmp.owSfrAJFdb +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name-4738 ++ local user=postgres ++ local cluster=some-name-4738 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gzQMRY24Tj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ziADIV94XK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.gzQMRY24Tj +++ cat /tmp/tmp.ziADIV94XK +++ rm /tmp/tmp.gzQMRY24Tj /tmp/tmp.ziADIV94XK +++ return 0 ++ [[ xsome-name-4738 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1WH74zx5ir ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Pd8yqkYaMG +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.1WH74zx5ir +++ cat /tmp/tmp.Pd8yqkYaMG +++ rm /tmp/tmp.1WH74zx5ir /tmp/tmp.Pd8yqkYaMG +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.G3k6k9sjmk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1Xbr6hgTu2 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.G3k6k9sjmk +++ cat /tmp/tmp.1Xbr6hgTu2 +++ rm /tmp/tmp.G3k6k9sjmk /tmp/tmp.1Xbr6hgTu2 +++ return 0 ++ [[ xsome-name-4738-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.aGyTi4c4qi +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.87MRXbpc8S ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.aGyTi4c4qi ++++ cat /tmp/tmp.87MRXbpc8S ++++ rm /tmp/tmp.aGyTi4c4qi /tmp/tmp.87MRXbpc8S ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3HcYyz1Qpr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hCpfINhfiH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.3HcYyz1Qpr +++ cat /tmp/tmp.hCpfINhfiH +++ rm /tmp/tmp.3HcYyz1Qpr /tmp/tmp.hCpfINhfiH +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ck16ZAOIAK +++ mktemp ++ local LAST_ERR=/tmp/tmp.Glid15dlut ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Ck16ZAOIAK ++ cat /tmp/tmp.Glid15dlut ++ rm /tmp/tmp.Ck16ZAOIAK /tmp/tmp.Glid15dlut ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.zikRiVe1vK ++ mktemp + local LAST_ERR=/tmp/tmp.qcSo9HGEDd + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.zikRiVe1vK + cat /tmp/tmp.qcSo9HGEDd + rm /tmp/tmp.zikRiVe1vK /tmp/tmp.qcSo9HGEDd + return 0 ++ get_psql_user_pass postgres some-name-4738 ++ local user=postgres ++ local cluster=some-name-4738 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fvf2g6lKrL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OGGpFrAwk6 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.fvf2g6lKrL +++ cat /tmp/tmp.OGGpFrAwk6 +++ rm /tmp/tmp.fvf2g6lKrL /tmp/tmp.OGGpFrAwk6 +++ return 0 ++ [[ xsome-name-4738 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.l3U08V1eGM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0jn2LdrZEb +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.l3U08V1eGM +++ cat /tmp/tmp.0jn2LdrZEb +++ rm /tmp/tmp.l3U08V1eGM /tmp/tmp.0jn2LdrZEb +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.V98j3x59zw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iKunMp91uk +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.V98j3x59zw +++ cat /tmp/tmp.iKunMp91uk +++ rm /tmp/tmp.V98j3x59zw /tmp/tmp.iKunMp91uk +++ return 0 ++ [[ xsome-name-4738-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.xSSN1Wxzfn +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.UVUcthMHKk ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.xSSN1Wxzfn ++++ cat /tmp/tmp.UVUcthMHKk ++++ rm /tmp/tmp.xSSN1Wxzfn /tmp/tmp.UVUcthMHKk ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5gcpfDCJLE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MzBUVNAwM2 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.5gcpfDCJLE +++ cat /tmp/tmp.MzBUVNAwM2 +++ rm /tmp/tmp.5gcpfDCJLE /tmp/tmp.MzBUVNAwM2 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2tLP3gvV0w +++ mktemp ++ local LAST_ERR=/tmp/tmp.PMfcRrVofw ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.2tLP3gvV0w ++ cat /tmp/tmp.PMfcRrVofw ++ rm /tmp/tmp.2tLP3gvV0w /tmp/tmp.PMfcRrVofw ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.IjDegXe5N5 ++ mktemp + local LAST_ERR=/tmp/tmp.glSTEqRgKF + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.IjDegXe5N5 + cat /tmp/tmp.glSTEqRgKF + rm /tmp/tmp.IjDegXe5N5 /tmp/tmp.glSTEqRgKF + return 0 ++ get_psql_user_pass postgres some-name-4738 ++ local user=postgres ++ local cluster=some-name-4738 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2jevcCMT1F ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1MvA9JCS8e +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2jevcCMT1F +++ cat /tmp/tmp.1MvA9JCS8e +++ rm /tmp/tmp.2jevcCMT1F /tmp/tmp.1MvA9JCS8e +++ return 0 ++ [[ xsome-name-4738 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.V0xNqMq3xa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZfDaZ1tlSv +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.V0xNqMq3xa +++ cat /tmp/tmp.ZfDaZ1tlSv +++ rm /tmp/tmp.V0xNqMq3xa /tmp/tmp.ZfDaZ1tlSv +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ohVDfmKYkk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LS09HrTGJR +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ohVDfmKYkk +++ cat /tmp/tmp.LS09HrTGJR +++ rm /tmp/tmp.ohVDfmKYkk /tmp/tmp.LS09HrTGJR +++ return 0 ++ [[ xsome-name-4738-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.f79iBI4Ifi +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ccJcfb7AaT ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.f79iBI4Ifi ++++ cat /tmp/tmp.ccJcfb7AaT ++++ rm /tmp/tmp.f79iBI4Ifi /tmp/tmp.ccJcfb7AaT ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tloKRTlmcp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HjSNk04HCL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.tloKRTlmcp +++ cat /tmp/tmp.HjSNk04HCL +++ rm /tmp/tmp.tloKRTlmcp /tmp/tmp.HjSNk04HCL +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yLFpJw2n6O +++ mktemp ++ local LAST_ERR=/tmp/tmp.1b5JF1sk8e ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.yLFpJw2n6O ++ cat /tmp/tmp.1b5JF1sk8e ++ rm /tmp/tmp.yLFpJw2n6O /tmp/tmp.1b5JF1sk8e ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.xYebbf3krz ++ mktemp + local LAST_ERR=/tmp/tmp.HINFfmrexF + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.xYebbf3krz + cat /tmp/tmp.HINFfmrexF + rm /tmp/tmp.xYebbf3krz /tmp/tmp.HINFfmrexF + return 0 + sleep 10 +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.backup.schedule[0].name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.luPx7gGEwd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zOMqXesDug +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.backup.schedule[0].name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.luPx7gGEwd +++ cat /tmp/tmp.zOMqXesDug +++ rm /tmp/tmp.luPx7gGEwd /tmp/tmp.zOMqXesDug +++ return 0 ++ next_backup_after_sec some-name-4738-sch-backup ++ local conf_map_name=some-name-4738-sch-backup ++++ kubectl_bin get configmap/some-name-4738-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.METbqnFsi7 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.rX9PTtYxTX ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl get configmap/some-name-4738-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.METbqnFsi7 ++++ cat /tmp/tmp.rX9PTtYxTX ++++ rm /tmp/tmp.METbqnFsi7 /tmp/tmp.rX9PTtYxTX ++++ return 0 +++ /usr/bin/date +%s --date=2024-04-23T13:52:57Z --utc ++ local configmap_creation_time=1713880377 +++ kubectl_bin get configmap/some-name-4738-sch-backup -o 'jsonpath={.data.schedule}' +++ jq .schedule +++ grep -Eo '[0-9]+' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6C0KqR66JX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CRHKQvkcSv +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get configmap/some-name-4738-sch-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.6C0KqR66JX +++ cat /tmp/tmp.CRHKQvkcSv +++ rm /tmp/tmp.6C0KqR66JX /tmp/tmp.CRHKQvkcSv +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1713880377 300 +++ local start_time_sec=1713880377 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1713880677 +++ local next_interval_min_edge=177 +++ local next_backup_at_sec=1713880500 +++ echo 1713880500 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Tue, 23 Apr 2024 13:56:32 GMT ' ++++ /usr/bin/date --date 'Tue, 23 Apr 2024 13:56:32 GMT ' --utc +%s +++ echo 1713880592 ++ local result=-92 ++ [[ -92 -gt 0 ]] ++++ get_current_epoch_time +++++ curl -sI https://percona.com/ +++++ grep date +++++ /usr/bin/sed 's/date: //g' ++++ local 'curent_date=Tue, 23 Apr 2024 13:56:32 GMT ' +++++ /usr/bin/date --date 'Tue, 23 Apr 2024 13:56:32 GMT ' --utc +%s ++++ echo 1713880592 +++ calculate_next_backup_start 1713880592 300 +++ local start_time_sec=1713880592 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1713880892 +++ local next_interval_min_edge=92 +++ local next_backup_at_sec=1713880800 +++ echo 1713880800 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Tue, 23 Apr 2024 13:56:33 GMT ' ++++ /usr/bin/date --date 'Tue, 23 Apr 2024 13:56:33 GMT ' --utc +%s +++ echo 1713880593 ++ result=207 ++ [[ 207 -gt 0 ]] ++ echo 207 + sleep 207 + kubectl_bin wait job/some-name-4738-diff-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.aVpdn4GJum ++ mktemp + local LAST_ERR=/tmp/tmp.TzZgCh4F5f + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl wait job/some-name-4738-diff-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.aVpdn4GJum job.batch/some-name-4738-diff-sch-backup condition met + cat /tmp/tmp.TzZgCh4F5f + rm /tmp/tmp.aVpdn4GJum /tmp/tmp.TzZgCh4F5f + return 0 + restore some-name-4738 s3 diff + local cluster=some-name-4738 + local storage=s3 + local backup_type=diff + local restore_target= ++ get_psql_user_pass postgres some-name-4738 ++ local user=postgres ++ local cluster=some-name-4738 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hlwFz3pprQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Mm1zVJ0uHI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.hlwFz3pprQ +++ cat /tmp/tmp.Mm1zVJ0uHI +++ rm /tmp/tmp.hlwFz3pprQ /tmp/tmp.Mm1zVJ0uHI +++ return 0 ++ [[ xsome-name-4738 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Tf0fcHB7ZN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BF6PXMAPND +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Tf0fcHB7ZN +++ cat /tmp/tmp.BF6PXMAPND +++ rm /tmp/tmp.Tf0fcHB7ZN /tmp/tmp.BF6PXMAPND +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fJbQqCi8hV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W08Qzk2k42 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.fJbQqCi8hV +++ cat /tmp/tmp.W08Qzk2k42 +++ rm /tmp/tmp.fJbQqCi8hV /tmp/tmp.W08Qzk2k42 +++ return 0 ++ [[ xsome-name-4738-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.vFHQFDHiBc +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.zWfEZuqbSL ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.vFHQFDHiBc ++++ cat /tmp/tmp.zWfEZuqbSL ++++ rm /tmp/tmp.vFHQFDHiBc /tmp/tmp.zWfEZuqbSL ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Czgp2gc0V9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HLHWcH93WN +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Czgp2gc0V9 +++ cat /tmp/tmp.HLHWcH93WN +++ rm /tmp/tmp.Czgp2gc0V9 /tmp/tmp.HLHWcH93WN +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TaGbM42dYf +++ mktemp ++ local LAST_ERR=/tmp/tmp.WGJzussDny ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.TaGbM42dYf ++ cat /tmp/tmp.WGJzussDny ++ rm /tmp/tmp.TaGbM42dYf /tmp/tmp.WGJzussDny ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.UlIAz9Ngzg ++ mktemp + local LAST_ERR=/tmp/tmp.4NQsgiLXlO + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.UlIAz9Ngzg + cat /tmp/tmp.4NQsgiLXlO + rm /tmp/tmp.UlIAz9Ngzg /tmp/tmp.4NQsgiLXlO + return 0 ++ get_psql_user_pass postgres some-name-4738 ++ local user=postgres ++ local cluster=some-name-4738 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1lYKSMaCNx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vnZwQ6ndG2 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.1lYKSMaCNx +++ cat /tmp/tmp.vnZwQ6ndG2 +++ rm /tmp/tmp.1lYKSMaCNx /tmp/tmp.vnZwQ6ndG2 +++ return 0 ++ [[ xsome-name-4738 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EyHmCLhQOJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NxnqtkEDmt +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.EyHmCLhQOJ +++ cat /tmp/tmp.NxnqtkEDmt +++ rm /tmp/tmp.EyHmCLhQOJ /tmp/tmp.NxnqtkEDmt +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ep2uAvXNNK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aSUMAezHTA +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ep2uAvXNNK +++ cat /tmp/tmp.aSUMAezHTA +++ rm /tmp/tmp.ep2uAvXNNK /tmp/tmp.aSUMAezHTA +++ return 0 ++ [[ xsome-name-4738-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ReBYyRfptO +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.BjohoqKIDg ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ReBYyRfptO ++++ cat /tmp/tmp.BjohoqKIDg ++++ rm /tmp/tmp.ReBYyRfptO /tmp/tmp.BjohoqKIDg ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HxhWJwGYf7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ojg5v4Xkqj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.HxhWJwGYf7 +++ cat /tmp/tmp.ojg5v4Xkqj +++ rm /tmp/tmp.HxhWJwGYf7 /tmp/tmp.ojg5v4Xkqj +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-4738.scheduled-backup-6083 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qRqyyPe7EF +++ mktemp ++ local LAST_ERR=/tmp/tmp.vC23Rd8x6v ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.qRqyyPe7EF ++ cat /tmp/tmp.vC23Rd8x6v ++ rm /tmp/tmp.qRqyyPe7EF /tmp/tmp.vC23Rd8x6v ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.dY9gVjEirl ++ mktemp + local LAST_ERR=/tmp/tmp.RpfJok4wPM + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.dY9gVjEirl + cat /tmp/tmp.RpfJok4wPM + rm /tmp/tmp.dY9gVjEirl /tmp/tmp.RpfJok4wPM + return 0 + '[' '!' -s /tmp/tmp.yIkqhaYiTU/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.yIkqhaYiTU/select-1.sql ++ get_psql_user_pass postgres some-name-4738 ++ local user=postgres ++ local cluster=some-name-4738 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CJxSYPyHgj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aVgh59K0zK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.CJxSYPyHgj +++ cat /tmp/tmp.aVgh59K0zK +++ rm /tmp/tmp.CJxSYPyHgj /tmp/tmp.aVgh59K0zK +++ return 0 ++ [[ xsome-name-4738 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DZCdip4FZD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oPtzCTxOgR +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.DZCdip4FZD +++ cat /tmp/tmp.oPtzCTxOgR +++ rm /tmp/tmp.DZCdip4FZD /tmp/tmp.oPtzCTxOgR +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9piFoufp88 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.m0Qx0Z0BP8 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.9piFoufp88 +++ cat /tmp/tmp.m0Qx0Z0BP8 +++ rm /tmp/tmp.9piFoufp88 /tmp/tmp.m0Qx0Z0BP8 +++ return 0 ++ [[ xsome-name-4738-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.XHqJONB5xs +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.5rGRLUjNQD ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.XHqJONB5xs ++++ cat /tmp/tmp.5rGRLUjNQD ++++ rm /tmp/tmp.XHqJONB5xs /tmp/tmp.5rGRLUjNQD ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.320yWfH5a8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.L8aPEeUwqu +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.320yWfH5a8 +++ cat /tmp/tmp.L8aPEeUwqu +++ rm /tmp/tmp.320yWfH5a8 /tmp/tmp.L8aPEeUwqu +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FhuBA3UtiH +++ mktemp ++ local LAST_ERR=/tmp/tmp.oBCWGAm7sd ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.FhuBA3UtiH ++ cat /tmp/tmp.oBCWGAm7sd ++ rm /tmp/tmp.FhuBA3UtiH /tmp/tmp.oBCWGAm7sd ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.eFHxy0Ovu1 ++ mktemp + local LAST_ERR=/tmp/tmp.qdK6e3iJTN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.eFHxy0Ovu1 + cat /tmp/tmp.qdK6e3iJTN + rm /tmp/tmp.eFHxy0Ovu1 /tmp/tmp.qdK6e3iJTN + return 0 ++ awk '{$1=$1;print}' +++ get_psql_user_pass postgres some-name-4738 +++ local user=postgres +++ local cluster=some-name-4738 +++ local namespace=scheduled-backup-6083 ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.zquUu14IXR +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.UNOD3782w2 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.zquUu14IXR ++++ cat /tmp/tmp.UNOD3782w2 ++++ rm /tmp/tmp.zquUu14IXR /tmp/tmp.UNOD3782w2 ++++ return 0 +++ [[ xsome-name-4738 == \x ]] +++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.FOiU0jI1pU +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.x6xKYdyE45 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.FOiU0jI1pU ++++ cat /tmp/tmp.x6xKYdyE45 ++++ rm /tmp/tmp.FOiU0jI1pU /tmp/tmp.x6xKYdyE45 ++++ return 0 ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.9JOWwT7qbI +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.S51QhMlaQm ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.9JOWwT7qbI ++++ cat /tmp/tmp.S51QhMlaQm ++++ rm /tmp/tmp.9JOWwT7qbI /tmp/tmp.S51QhMlaQm ++++ return 0 +++ [[ xsome-name-4738-users-secret == \x ]] ++++ base64 -d +++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.18ZTMxb3p5 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.BSfEJPIEcV +++++ local exit_status=0 +++++ local timeout=4 +++++ set +o errexit ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ set -o errexit +++++ cat /tmp/tmp.18ZTMxb3p5 +++++ cat /tmp/tmp.BSfEJPIEcV +++++ rm /tmp/tmp.18ZTMxb3p5 /tmp/tmp.BSfEJPIEcV +++++ return 0 ++++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.424Cv1hcW8 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.S4nL66zYuZ ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.424Cv1hcW8 ++++ cat /tmp/tmp.S4nL66zYuZ ++++ rm /tmp/tmp.424Cv1hcW8 /tmp/tmp.S4nL66zYuZ ++++ return 0 +++ echo postgres_password ++ run_psql '\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' postgres:postgres_password@some-name-4738.scheduled-backup-6083 ++ local 'command=\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' ++ local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 ++ local driver=postgres ++ local suffix=.svc.cluster.local +++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.614H3LSsPY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Qg8Jxrmo1p +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.614H3LSsPY +++ cat /tmp/tmp.Qg8Jxrmo1p +++ rm /tmp/tmp.614H3LSsPY /tmp/tmp.Qg8Jxrmo1p +++ return 0 ++ local client_container=pg-client-76df9df876-4grks ++ kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gRUPRV1HL5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5uBIjpQ5k9 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.gRUPRV1HL5 ++ cat /tmp/tmp.5uBIjpQ5k9 ++ rm /tmp/tmp.gRUPRV1HL5 /tmp/tmp.5uBIjpQ5k9 ++ return 0 + restore_target=773 ++ get_psql_user_pass postgres some-name-4738 ++ local user=postgres ++ local cluster=some-name-4738 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CuN5iOEbzB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NfjNKDJf9S +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.CuN5iOEbzB +++ cat /tmp/tmp.NfjNKDJf9S +++ rm /tmp/tmp.CuN5iOEbzB /tmp/tmp.NfjNKDJf9S +++ return 0 ++ [[ xsome-name-4738 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.v8Ps2B5XFX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MLS3lSk0z8 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.v8Ps2B5XFX +++ cat /tmp/tmp.MLS3lSk0z8 +++ rm /tmp/tmp.v8Ps2B5XFX /tmp/tmp.MLS3lSk0z8 +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BlaTNEE8TR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RQu56Gylua +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.BlaTNEE8TR +++ cat /tmp/tmp.RQu56Gylua +++ rm /tmp/tmp.BlaTNEE8TR /tmp/tmp.RQu56Gylua +++ return 0 ++ [[ xsome-name-4738-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.CARLzg77so +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.4rMo1ElNOL ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.CARLzg77so ++++ cat /tmp/tmp.4rMo1ElNOL ++++ rm /tmp/tmp.CARLzg77so /tmp/tmp.4rMo1ElNOL ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.F7bmS7dIwC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gLOywUxuGM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.F7bmS7dIwC +++ cat /tmp/tmp.gLOywUxuGM +++ rm /tmp/tmp.F7bmS7dIwC /tmp/tmp.gLOywUxuGM +++ return 0 ++ echo postgres_password + run_psql 'DROP DATABASE myapp' postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local 'command=DROP DATABASE myapp' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qnBPaBsxhN +++ mktemp ++ local LAST_ERR=/tmp/tmp.IR4nKgep4T ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.qnBPaBsxhN ++ cat /tmp/tmp.IR4nKgep4T ++ rm /tmp/tmp.qnBPaBsxhN /tmp/tmp.IR4nKgep4T ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.xJ2iKl93HN ++ mktemp + local LAST_ERR=/tmp/tmp.F9gWmv3Rgj + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.xJ2iKl93HN + cat /tmp/tmp.F9gWmv3Rgj + rm /tmp/tmp.xJ2iKl93HN /tmp/tmp.F9gWmv3Rgj + return 0 + run_restore some-name-4738 diff s3 773 xid + local cluster=some-name-4738 + local rstr_prefix=diff + local storage=s3 + local target=773 + local type=xid + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/conf/restore.yml metadata.labels.pg-cluster some-name-4738 + yq w - metadata.name diff-some-name-4738 + yq w - spec.namespace scheduled-backup-6083 + yq w - spec.parameters.backrest-restore-cluster some-name-4738 + yq w - spec.parameters.backrest-storage-type s3 + yq w - spec.name diff-some-name-4738 + [[ 773 != \n\u\l\l ]] + yq w /tmp/tmp.yIkqhaYiTU/restore.yml --style=single spec.parameters.backrest-pitr-target 773 + yq w --style=single -- - spec.parameters.backrest-restore-opts --type=xid + mv /tmp/tmp.yIkqhaYiTU/restore.pitr.yml /tmp/tmp.yIkqhaYiTU/restore.yml + kubectl apply -f /tmp/tmp.yIkqhaYiTU/restore.yml pgtask.pg.percona.com/diff-some-name-4738 created + wait_bootstrap_completeness some-name-4738 + local cluster=some-name-4738 + wait_job_completion some-name-4738-bootstrap + local job=some-name-4738-bootstrap + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' some-name-4738-bootstrap+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_deployment some-name-4738-backrest-shared-repo + local name=some-name-4738-backrest-shared-repo + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-4738-backrest-shared-repo. + wait_deployment some-name-4738 + local name=some-name-4738 + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-4738.... + wait_job_completion some-name-4738-stanza-create false + local job=some-name-4738-stanza-create + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' some-name-4738-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name-4738 + local job=backrest-backup-some-name-4738 + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-4738+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 48 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-4738 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo ++ get_psql_user_pass postgres some-name-4738 ++ local user=postgres ++ local cluster=some-name-4738 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TQ9JUjBr7J ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kTfzGa2SQl +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TQ9JUjBr7J +++ cat /tmp/tmp.kTfzGa2SQl +++ rm /tmp/tmp.TQ9JUjBr7J /tmp/tmp.kTfzGa2SQl +++ return 0 ++ [[ xsome-name-4738 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.x9JPH9BuCS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oX88lPCuPz +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-4738 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.x9JPH9BuCS +++ cat /tmp/tmp.oX88lPCuPz +++ rm /tmp/tmp.x9JPH9BuCS /tmp/tmp.oX88lPCuPz +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PLNAEz3PEn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iZSBOzzs0l +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.PLNAEz3PEn +++ cat /tmp/tmp.iZSBOzzs0l +++ rm /tmp/tmp.PLNAEz3PEn /tmp/tmp.iZSBOzzs0l +++ return 0 ++ [[ xsome-name-4738-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ehJzh2R4Fs +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.06Ce6Kqu0F ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-4738 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ehJzh2R4Fs ++++ cat /tmp/tmp.06Ce6Kqu0F ++++ rm /tmp/tmp.ehJzh2R4Fs /tmp/tmp.06Ce6Kqu0F ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XwRGoxtPJo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KqN1FDuViY +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-4738-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.XwRGoxtPJo +++ cat /tmp/tmp.KqN1FDuViY +++ rm /tmp/tmp.XwRGoxtPJo /tmp/tmp.KqN1FDuViY +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-4738.scheduled-backup-6083 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-4738.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V2xOiSyd8t +++ mktemp ++ local LAST_ERR=/tmp/tmp.UrP87peO7l ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.V2xOiSyd8t ++ cat /tmp/tmp.UrP87peO7l ++ rm /tmp/tmp.V2xOiSyd8t /tmp/tmp.UrP87peO7l ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.dz08CpsakG ++ mktemp + local LAST_ERR=/tmp/tmp.8EXrqvJAYR + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-4738.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.dz08CpsakG + cat /tmp/tmp.8EXrqvJAYR + rm /tmp/tmp.dz08CpsakG /tmp/tmp.8EXrqvJAYR + return 0 + '[' '!' -s /tmp/tmp.yIkqhaYiTU/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.yIkqhaYiTU/select-1.sql + delete_cluster_gracefully some-name-4738 + local cluster=some-name-4738 + kubectl_bin delete perconapgcluster/some-name-4738 ++ mktemp + local LAST_OUT=/tmp/tmp.c12sEQKJb8 ++ mktemp + local LAST_ERR=/tmp/tmp.88BqNkkP6e + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete perconapgcluster/some-name-4738 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.c12sEQKJb8 perconapgcluster.pg.percona.com "some-name-4738" deleted + cat /tmp/tmp.88BqNkkP6e + rm /tmp/tmp.c12sEQKJb8 /tmp/tmp.88BqNkkP6e + return 0 ++ kubectl_bin -n scheduled-backup-6083 get jobs --selector=pg-cluster=some-name-4738,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.maC0JUzekH +++ mktemp ++ local LAST_ERR=/tmp/tmp.SavwLWxjZX ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get jobs --selector=pg-cluster=some-name-4738,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.maC0JUzekH ++ cat /tmp/tmp.SavwLWxjZX ++ rm /tmp/tmp.maC0JUzekH /tmp/tmp.SavwLWxjZX ++ return 0 + rmjob_name=some-name-4738-rmdata-nesv + wait_job_completion some-name-4738-rmdata-nesv false + local job=some-name-4738-rmdata-nesv + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' some-name-4738-rmdata-nesv+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 48 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 49 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 50 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 51 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 52 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-4738-rmdata-nesv -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + desc 'Testing incr backup/restore from s3' + set +o xtrace ----------------------------------------------------------------------------------- Testing incr backup/restore from s3 ----------------------------------------------------------------------------------- + cluster=some-name-21191 + apply_configs some-name-21191 + local cluster=some-name-21191 + set +x secret/some-name-21191-backrest-repo-config created + yq w - metadata.name some-name-21191-custom-config + kubectl_bin apply -f - + yq r /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/conf/custom-config.yml ++ mktemp + local LAST_OUT=/tmp/tmp.5TRZJdP1FN ++ mktemp + local LAST_ERR=/tmp/tmp.4BlP2RPv20 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5TRZJdP1FN configmap/some-name-21191-custom-config created + cat /tmp/tmp.4BlP2RPv20 + rm /tmp/tmp.5TRZJdP1FN /tmp/tmp.4BlP2RPv20 + return 0 + spinup_pgcluster some-name-21191 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml s3 some-name-21191-custom-config false true incr + local cluster=some-name-21191 + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local backup=s3 + local custom_config=some-name-21191-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=incr + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + local namespace=scheduled-backup-6083 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name-21191 scheduled-backup-6083 + local cluster=some-name-21191 + local namespace=scheduled-backup-6083 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + yq w - metadata.name some-name-21191-users-secret + yq w - stringData.some-name some-name_pass + kubectl_bin -n scheduled-backup-6083 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AYBOyxERID ++ mktemp + local LAST_ERR=/tmp/tmp.lCfNLKNl1K + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.AYBOyxERID secret/some-name-21191-users-secret created + cat /tmp/tmp.lCfNLKNl1K + rm /tmp/tmp.AYBOyxERID /tmp/tmp.lCfNLKNl1K + return 0 + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + yq w - metadata.name some-name-21191-pmm-secret + kubectl_bin -n scheduled-backup-6083 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.VsHT0iTB9p ++ mktemp + local LAST_ERR=/tmp/tmp.n2dggC5XN6 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.VsHT0iTB9p secret/some-name-21191-pmm-secret created + cat /tmp/tmp.n2dggC5XN6 + rm /tmp/tmp.VsHT0iTB9p /tmp/tmp.n2dggC5XN6 + return 0 + kubectl_bin -n scheduled-backup-6083 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.jMOUGAQK8v ++ mktemp + local LAST_ERR=/tmp/tmp.K433Bpofxs + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.jMOUGAQK8v deployment.apps/pg-client unchanged + cat /tmp/tmp.K433Bpofxs + rm /tmp/tmp.jMOUGAQK8v /tmp/tmp.K433Bpofxs + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name-21191 s3 false false some-name-21191-custom-config false true incr false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local name=some-name-21191 + local backup=s3 + local restore_from=false + local restore_opts=false + local custom_config=some-name-21191-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=incr + local tls_only=false + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name-21191 + yq w - metadata.labels.crunchy-pgha-scope some-name-21191 + yq w - spec.name some-name-21191 + yq w - spec.database some-name-21191 + yq w - spec.keepData false + yq w - spec.keepBackups false + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.pmm.image perconalab/pmm-client:2.41.0 + yq w - metadata.labels.pg-cluster some-name-21191 + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.clustername some-name-21191 + yq w - spec.user some-name + yq w - spec.secretsName some-name-21191-users-secret + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - metadata.name some-name-21191 + yq w - metadata.labels.deployment-name some-name-21191 + yq w - metadata.labels.name some-name-21191 + case ${podAntiAffinity} in + case ${backup} in + yq w /tmp/tmp.yIkqhaYiTU/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + [[ true == \t\r\u\e ]] + yq w /tmp/tmp.yIkqhaYiTU/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w - 'spec.backup.schedule[0].schedule' '*/5 * * * *' + yq w - 'spec.backup.schedule[0].type' incr + yq w - 'spec.backup.schedule[0].storage' my-s3 + yq w - 'spec.backup.schedule[0].name' sch-backup + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + mv /tmp/tmp.yIkqhaYiTU/cr.backup.yaml /tmp/tmp.yIkqhaYiTU/cr.yaml + [[ false != \f\a\l\s\e ]] + [[ some-name-21191-custom-config != \f\a\l\s\e ]] + yq w -i /tmp/tmp.yIkqhaYiTU/cr.yaml spec.pgPrimary.customconfig some-name-21191-custom-config + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.yIkqhaYiTU/cr.yaml -n scheduled-backup-6083 ++ mktemp + local LAST_OUT=/tmp/tmp.2uVw3Rrpbd ++ mktemp + local LAST_ERR=/tmp/tmp.FHcqvalTzw + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f /tmp/tmp.yIkqhaYiTU/cr.yaml -n scheduled-backup-6083 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2uVw3Rrpbd perconapgcluster.pg.percona.com/some-name-21191 created + cat /tmp/tmp.FHcqvalTzw + rm /tmp/tmp.2uVw3Rrpbd /tmp/tmp.FHcqvalTzw + return 0 + wait_deployment some-name-21191-backrest-shared-repo + local name=some-name-21191-backrest-shared-repo + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-21191-backrest-shared-repo.. + wait_deployment some-name-21191 + local name=some-name-21191 + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-21191..... + [[ s3 == \g\c\s\+* ]] + wait_job_completion some-name-21191-stanza-create false + local job=some-name-21191-stanza-create + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.metadata.name}' some-name-21191-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name-21191 + local job=backrest-backup-some-name-21191 + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-21191+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name-21191 + cluster_name=some-name-21191 + wait_cluster_status some-name-21191 'pgcluster Initialized' + local cluster_name=some-name-21191 + local 'target_status=pgcluster Initialized' + local target_namespace=scheduled-backup-6083 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9tkFs2Vzd6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.UwgAnkOvNd ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.9tkFs2Vzd6 ++ cat /tmp/tmp.UwgAnkOvNd ++ rm /tmp/tmp.9tkFs2Vzd6 /tmp/tmp.UwgAnkOvNd ++ return 0 + status_query='kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MAoZDxav9B ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KdJdKxLAej +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.MAoZDxav9B +++ cat /tmp/tmp.KdJdKxLAej +++ rm /tmp/tmp.MAoZDxav9B /tmp/tmp.KdJdKxLAej +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name-21191 ++ local user=postgres ++ local cluster=some-name-21191 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eifa9z0Swn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IO4TRmChk4 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.eifa9z0Swn +++ cat /tmp/tmp.IO4TRmChk4 +++ rm /tmp/tmp.eifa9z0Swn /tmp/tmp.IO4TRmChk4 +++ return 0 ++ [[ xsome-name-21191 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TaKYlh87vf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oMeZzIdhyo +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TaKYlh87vf +++ cat /tmp/tmp.oMeZzIdhyo +++ rm /tmp/tmp.TaKYlh87vf /tmp/tmp.oMeZzIdhyo +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7VeB1Hpz2S ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cNDTT1Q9qd +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7VeB1Hpz2S +++ cat /tmp/tmp.cNDTT1Q9qd +++ rm /tmp/tmp.7VeB1Hpz2S /tmp/tmp.cNDTT1Q9qd +++ return 0 ++ [[ xsome-name-21191-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.zX2kzudSj2 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.O6vi6tLMCB ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.zX2kzudSj2 ++++ cat /tmp/tmp.O6vi6tLMCB ++++ rm /tmp/tmp.zX2kzudSj2 /tmp/tmp.O6vi6tLMCB ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gBLILBuRPt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4nNjommEmd +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.gBLILBuRPt +++ cat /tmp/tmp.4nNjommEmd +++ rm /tmp/tmp.gBLILBuRPt /tmp/tmp.4nNjommEmd +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4tkX8WyLYN +++ mktemp ++ local LAST_ERR=/tmp/tmp.9ete3LrWcd ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.4tkX8WyLYN ++ cat /tmp/tmp.9ete3LrWcd ++ rm /tmp/tmp.4tkX8WyLYN /tmp/tmp.9ete3LrWcd ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.J5pNOfyHRN ++ mktemp + local LAST_ERR=/tmp/tmp.qdpbIR3vLc + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.J5pNOfyHRN + cat /tmp/tmp.qdpbIR3vLc + rm /tmp/tmp.J5pNOfyHRN /tmp/tmp.qdpbIR3vLc + return 0 ++ get_psql_user_pass postgres some-name-21191 ++ local user=postgres ++ local cluster=some-name-21191 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.j8cbOlsKYS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W4szq8SIvB +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.j8cbOlsKYS +++ cat /tmp/tmp.W4szq8SIvB +++ rm /tmp/tmp.j8cbOlsKYS /tmp/tmp.W4szq8SIvB +++ return 0 ++ [[ xsome-name-21191 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VXBKBnMjGJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cENrF58AQX +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.VXBKBnMjGJ +++ cat /tmp/tmp.cENrF58AQX +++ rm /tmp/tmp.VXBKBnMjGJ /tmp/tmp.cENrF58AQX +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cOAEdzFgjz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sKdPNjBOv1 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.cOAEdzFgjz +++ cat /tmp/tmp.sKdPNjBOv1 +++ rm /tmp/tmp.cOAEdzFgjz /tmp/tmp.sKdPNjBOv1 +++ return 0 ++ [[ xsome-name-21191-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.aKvBC2MHT7 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.L90RIUgpXL ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.aKvBC2MHT7 ++++ cat /tmp/tmp.L90RIUgpXL ++++ rm /tmp/tmp.aKvBC2MHT7 /tmp/tmp.L90RIUgpXL ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zErTZGO3jV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rkKBzEXmAQ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zErTZGO3jV +++ cat /tmp/tmp.rkKBzEXmAQ +++ rm /tmp/tmp.zErTZGO3jV /tmp/tmp.rkKBzEXmAQ +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N9InjCIlpl +++ mktemp ++ local LAST_ERR=/tmp/tmp.cjAPBXFJH2 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.N9InjCIlpl ++ cat /tmp/tmp.cjAPBXFJH2 ++ rm /tmp/tmp.N9InjCIlpl /tmp/tmp.cjAPBXFJH2 ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.fg0iPil1G8 ++ mktemp + local LAST_ERR=/tmp/tmp.siKla91VVU + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.fg0iPil1G8 + cat /tmp/tmp.siKla91VVU + rm /tmp/tmp.fg0iPil1G8 /tmp/tmp.siKla91VVU + return 0 ++ get_psql_user_pass postgres some-name-21191 ++ local user=postgres ++ local cluster=some-name-21191 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vsD0qhSXkb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uUZeOqBf8h +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.vsD0qhSXkb +++ cat /tmp/tmp.uUZeOqBf8h +++ rm /tmp/tmp.vsD0qhSXkb /tmp/tmp.uUZeOqBf8h +++ return 0 ++ [[ xsome-name-21191 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GEYDzb96oA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.o1s71FSLpy +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.GEYDzb96oA +++ cat /tmp/tmp.o1s71FSLpy +++ rm /tmp/tmp.GEYDzb96oA /tmp/tmp.o1s71FSLpy +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DdxnuQZ21I ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OYTQjKEyi8 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.DdxnuQZ21I +++ cat /tmp/tmp.OYTQjKEyi8 +++ rm /tmp/tmp.DdxnuQZ21I /tmp/tmp.OYTQjKEyi8 +++ return 0 ++ [[ xsome-name-21191-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.y5oNkn6h2S +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ekAvxdE2yY ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.y5oNkn6h2S ++++ cat /tmp/tmp.ekAvxdE2yY ++++ rm /tmp/tmp.y5oNkn6h2S /tmp/tmp.ekAvxdE2yY ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oLqmpMXOhP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5f3X68fZAl +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.oLqmpMXOhP +++ cat /tmp/tmp.5f3X68fZAl +++ rm /tmp/tmp.oLqmpMXOhP /tmp/tmp.5f3X68fZAl +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2pyiRNoPQa +++ mktemp ++ local LAST_ERR=/tmp/tmp.7iW6rSCTW4 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.2pyiRNoPQa ++ cat /tmp/tmp.7iW6rSCTW4 ++ rm /tmp/tmp.2pyiRNoPQa /tmp/tmp.7iW6rSCTW4 ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.nbohaptgoH ++ mktemp + local LAST_ERR=/tmp/tmp.y1eNJgaQ0m + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.nbohaptgoH + cat /tmp/tmp.y1eNJgaQ0m + rm /tmp/tmp.nbohaptgoH /tmp/tmp.y1eNJgaQ0m + return 0 + sleep 10 +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.backup.schedule[0].name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VAIweko7wr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LSEq27ODcN +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.backup.schedule[0].name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.VAIweko7wr +++ cat /tmp/tmp.LSEq27ODcN +++ rm /tmp/tmp.VAIweko7wr /tmp/tmp.LSEq27ODcN +++ return 0 ++ next_backup_after_sec some-name-21191-sch-backup ++ local conf_map_name=some-name-21191-sch-backup ++++ kubectl_bin get configmap/some-name-21191-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ioLtqJFJiO +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.1dGw6tKY5w ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl get configmap/some-name-21191-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ioLtqJFJiO ++++ cat /tmp/tmp.1dGw6tKY5w ++++ rm /tmp/tmp.ioLtqJFJiO /tmp/tmp.1dGw6tKY5w ++++ return 0 +++ /usr/bin/date +%s --date=2024-04-23T14:08:17Z --utc ++ local configmap_creation_time=1713881297 +++ kubectl_bin get configmap/some-name-21191-sch-backup -o 'jsonpath={.data.schedule}' +++ jq .schedule +++ grep -Eo '[0-9]+' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ChZj1XJ1Wr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NXuR8cScm1 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get configmap/some-name-21191-sch-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ChZj1XJ1Wr +++ cat /tmp/tmp.NXuR8cScm1 +++ rm /tmp/tmp.ChZj1XJ1Wr /tmp/tmp.NXuR8cScm1 +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1713881297 300 +++ local start_time_sec=1713881297 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1713881597 +++ local next_interval_min_edge=197 +++ local next_backup_at_sec=1713881400 +++ echo 1713881400 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ /usr/bin/sed 's/date: //g' ++++ grep date +++ local 'curent_date=Tue, 23 Apr 2024 14:11:57 GMT ' ++++ /usr/bin/date --date 'Tue, 23 Apr 2024 14:11:57 GMT ' --utc +%s +++ echo 1713881517 ++ local result=-117 ++ [[ -117 -gt 0 ]] ++++ get_current_epoch_time +++++ curl -sI https://percona.com/ +++++ grep date +++++ /usr/bin/sed 's/date: //g' ++++ local 'curent_date=Tue, 23 Apr 2024 14:11:58 GMT ' +++++ /usr/bin/date --date 'Tue, 23 Apr 2024 14:11:58 GMT ' --utc +%s ++++ echo 1713881518 +++ calculate_next_backup_start 1713881518 300 +++ local start_time_sec=1713881518 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1713881818 +++ local next_interval_min_edge=118 +++ local next_backup_at_sec=1713881700 +++ echo 1713881700 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Tue, 23 Apr 2024 14:11:58 GMT ' ++++ /usr/bin/date --date 'Tue, 23 Apr 2024 14:11:58 GMT ' --utc +%s +++ echo 1713881518 ++ result=182 ++ [[ 182 -gt 0 ]] ++ echo 182 + sleep 182 + kubectl_bin wait job/some-name-21191-incr-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.1OH4GJQfvz ++ mktemp + local LAST_ERR=/tmp/tmp.5oM4oegP5v + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl wait job/some-name-21191-incr-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.1OH4GJQfvz job.batch/some-name-21191-incr-sch-backup condition met + cat /tmp/tmp.5oM4oegP5v + rm /tmp/tmp.1OH4GJQfvz /tmp/tmp.5oM4oegP5v + return 0 + restore some-name-21191 s3 incr + local cluster=some-name-21191 + local storage=s3 + local backup_type=incr + local restore_target= ++ get_psql_user_pass postgres some-name-21191 ++ local user=postgres ++ local cluster=some-name-21191 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cY5pTR4cWr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9d2MyyvaRz +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.cY5pTR4cWr +++ cat /tmp/tmp.9d2MyyvaRz +++ rm /tmp/tmp.cY5pTR4cWr /tmp/tmp.9d2MyyvaRz +++ return 0 ++ [[ xsome-name-21191 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.d7yVwo8Ha4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wDGy878gtr +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.d7yVwo8Ha4 +++ cat /tmp/tmp.wDGy878gtr +++ rm /tmp/tmp.d7yVwo8Ha4 /tmp/tmp.wDGy878gtr +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2dGuYoBsPu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.supIEuN3Pr +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2dGuYoBsPu +++ cat /tmp/tmp.supIEuN3Pr +++ rm /tmp/tmp.2dGuYoBsPu /tmp/tmp.supIEuN3Pr +++ return 0 ++ [[ xsome-name-21191-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.3VIbPnO6Yl +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.WVGgRhrQyp ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.3VIbPnO6Yl ++++ cat /tmp/tmp.WVGgRhrQyp ++++ rm /tmp/tmp.3VIbPnO6Yl /tmp/tmp.WVGgRhrQyp ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.axpbosaGuh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xsAP0K57XZ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.axpbosaGuh +++ cat /tmp/tmp.xsAP0K57XZ +++ rm /tmp/tmp.axpbosaGuh /tmp/tmp.xsAP0K57XZ +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zYWfm3v7Re +++ mktemp ++ local LAST_ERR=/tmp/tmp.5PAGn8fR3n ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.zYWfm3v7Re ++ cat /tmp/tmp.5PAGn8fR3n ++ rm /tmp/tmp.zYWfm3v7Re /tmp/tmp.5PAGn8fR3n ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.T0pTBmXjn9 ++ mktemp + local LAST_ERR=/tmp/tmp.G2z6bbApQ0 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.T0pTBmXjn9 + cat /tmp/tmp.G2z6bbApQ0 + rm /tmp/tmp.T0pTBmXjn9 /tmp/tmp.G2z6bbApQ0 + return 0 ++ get_psql_user_pass postgres some-name-21191 ++ local user=postgres ++ local cluster=some-name-21191 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.P6Q1PxgXzL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mVd1u3raAz +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.P6Q1PxgXzL +++ cat /tmp/tmp.mVd1u3raAz +++ rm /tmp/tmp.P6Q1PxgXzL /tmp/tmp.mVd1u3raAz +++ return 0 ++ [[ xsome-name-21191 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Md16fnsPjL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JurO02NY3W +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Md16fnsPjL +++ cat /tmp/tmp.JurO02NY3W +++ rm /tmp/tmp.Md16fnsPjL /tmp/tmp.JurO02NY3W +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BhSGFmnnQe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XbwUAsFsDj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.BhSGFmnnQe +++ cat /tmp/tmp.XbwUAsFsDj +++ rm /tmp/tmp.BhSGFmnnQe /tmp/tmp.XbwUAsFsDj +++ return 0 ++ [[ xsome-name-21191-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.pkIhfgnFGx +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2iT0t10pCg ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.pkIhfgnFGx ++++ cat /tmp/tmp.2iT0t10pCg ++++ rm /tmp/tmp.pkIhfgnFGx /tmp/tmp.2iT0t10pCg ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DSncZJxlwi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.p4VhLQSGu0 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.DSncZJxlwi +++ cat /tmp/tmp.p4VhLQSGu0 +++ rm /tmp/tmp.DSncZJxlwi /tmp/tmp.p4VhLQSGu0 +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-21191.scheduled-backup-6083 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Al7KEoh5bw +++ mktemp ++ local LAST_ERR=/tmp/tmp.4NxSQXhuIW ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Al7KEoh5bw ++ cat /tmp/tmp.4NxSQXhuIW ++ rm /tmp/tmp.Al7KEoh5bw /tmp/tmp.4NxSQXhuIW ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.qzVHjPChJc ++ mktemp + local LAST_ERR=/tmp/tmp.yKFNMfqxyY + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.qzVHjPChJc + cat /tmp/tmp.yKFNMfqxyY + rm /tmp/tmp.qzVHjPChJc /tmp/tmp.yKFNMfqxyY + return 0 + '[' '!' -s /tmp/tmp.yIkqhaYiTU/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.yIkqhaYiTU/select-1.sql ++ get_psql_user_pass postgres some-name-21191 ++ local user=postgres ++ local cluster=some-name-21191 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.P4wLqeMDR5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vle4u0uAXf +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.P4wLqeMDR5 +++ cat /tmp/tmp.vle4u0uAXf +++ rm /tmp/tmp.P4wLqeMDR5 /tmp/tmp.vle4u0uAXf +++ return 0 ++ [[ xsome-name-21191 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.btl679soAx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AEBCUesYJ1 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.btl679soAx +++ cat /tmp/tmp.AEBCUesYJ1 +++ rm /tmp/tmp.btl679soAx /tmp/tmp.AEBCUesYJ1 +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pJbN99gay0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eoLEdKXLWE +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.pJbN99gay0 +++ cat /tmp/tmp.eoLEdKXLWE +++ rm /tmp/tmp.pJbN99gay0 /tmp/tmp.eoLEdKXLWE +++ return 0 ++ [[ xsome-name-21191-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.bWTRUmSMQV +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.CN1tIdZFZL ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.bWTRUmSMQV ++++ cat /tmp/tmp.CN1tIdZFZL ++++ rm /tmp/tmp.bWTRUmSMQV /tmp/tmp.CN1tIdZFZL ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0YS8XktPyk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zzLwgW2PSp +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.0YS8XktPyk +++ cat /tmp/tmp.zzLwgW2PSp +++ rm /tmp/tmp.0YS8XktPyk /tmp/tmp.zzLwgW2PSp +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pKl4p6APPW +++ mktemp ++ local LAST_ERR=/tmp/tmp.u3Sd0dhNwg ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.pKl4p6APPW ++ cat /tmp/tmp.u3Sd0dhNwg ++ rm /tmp/tmp.pKl4p6APPW /tmp/tmp.u3Sd0dhNwg ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.QZJFJJEWtm ++ mktemp + local LAST_ERR=/tmp/tmp.U3zJxvEL4a + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.QZJFJJEWtm + cat /tmp/tmp.U3zJxvEL4a + rm /tmp/tmp.QZJFJJEWtm /tmp/tmp.U3zJxvEL4a + return 0 ++ awk '{$1=$1;print}' +++ get_psql_user_pass postgres some-name-21191 +++ local user=postgres +++ local cluster=some-name-21191 +++ local namespace=scheduled-backup-6083 ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.u97LMiPVOt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.pupHoqxc6b ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.u97LMiPVOt ++++ cat /tmp/tmp.pupHoqxc6b ++++ rm /tmp/tmp.u97LMiPVOt /tmp/tmp.pupHoqxc6b ++++ return 0 +++ [[ xsome-name-21191 == \x ]] +++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.odkwHB1oo5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.shLihbJkaq ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.odkwHB1oo5 ++++ cat /tmp/tmp.shLihbJkaq ++++ rm /tmp/tmp.odkwHB1oo5 /tmp/tmp.shLihbJkaq ++++ return 0 ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.5Sru09Fhft +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.4ZzuH7oMGX ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.5Sru09Fhft ++++ cat /tmp/tmp.4ZzuH7oMGX ++++ rm /tmp/tmp.5Sru09Fhft /tmp/tmp.4ZzuH7oMGX ++++ return 0 +++ [[ xsome-name-21191-users-secret == \x ]] ++++ base64 -d +++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.c5xyaiJFpr ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.4YYRNfVF19 +++++ local exit_status=0 +++++ local timeout=4 +++++ set +o errexit ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ set -o errexit +++++ cat /tmp/tmp.c5xyaiJFpr +++++ cat /tmp/tmp.4YYRNfVF19 +++++ rm /tmp/tmp.c5xyaiJFpr /tmp/tmp.4YYRNfVF19 +++++ return 0 ++++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.04R9LwiZVU +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.MfKOwkA7aS ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.04R9LwiZVU ++++ cat /tmp/tmp.MfKOwkA7aS ++++ rm /tmp/tmp.04R9LwiZVU /tmp/tmp.MfKOwkA7aS ++++ return 0 +++ echo postgres_password ++ run_psql '\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' postgres:postgres_password@some-name-21191.scheduled-backup-6083 ++ local 'command=\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' ++ local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 ++ local driver=postgres ++ local suffix=.svc.cluster.local +++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.S47e7sgnr2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.klenzfTQrV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.S47e7sgnr2 +++ cat /tmp/tmp.klenzfTQrV +++ rm /tmp/tmp.S47e7sgnr2 /tmp/tmp.klenzfTQrV +++ return 0 ++ local client_container=pg-client-76df9df876-4grks ++ kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CNslmuM1oR +++ mktemp ++ local LAST_ERR=/tmp/tmp.2XA69iHFCT ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.CNslmuM1oR ++ cat /tmp/tmp.2XA69iHFCT ++ rm /tmp/tmp.CNslmuM1oR /tmp/tmp.2XA69iHFCT ++ return 0 + restore_target=773 ++ get_psql_user_pass postgres some-name-21191 ++ local user=postgres ++ local cluster=some-name-21191 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SvHqHL7Gg4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iSE6P7Damz +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.SvHqHL7Gg4 +++ cat /tmp/tmp.iSE6P7Damz +++ rm /tmp/tmp.SvHqHL7Gg4 /tmp/tmp.iSE6P7Damz +++ return 0 ++ [[ xsome-name-21191 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.L0Arw0lIek ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kF8i3Nw8BK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.L0Arw0lIek +++ cat /tmp/tmp.kF8i3Nw8BK +++ rm /tmp/tmp.L0Arw0lIek /tmp/tmp.kF8i3Nw8BK +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.M6cwHTkYrs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.53C64vloGZ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.M6cwHTkYrs +++ cat /tmp/tmp.53C64vloGZ +++ rm /tmp/tmp.M6cwHTkYrs /tmp/tmp.53C64vloGZ +++ return 0 ++ [[ xsome-name-21191-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.eAyGL3ofPo +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.IDzDhIA617 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.eAyGL3ofPo ++++ cat /tmp/tmp.IDzDhIA617 ++++ rm /tmp/tmp.eAyGL3ofPo /tmp/tmp.IDzDhIA617 ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bv0NtWoPbR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pS7FMYXBiy +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bv0NtWoPbR +++ cat /tmp/tmp.pS7FMYXBiy +++ rm /tmp/tmp.bv0NtWoPbR /tmp/tmp.pS7FMYXBiy +++ return 0 ++ echo postgres_password + run_psql 'DROP DATABASE myapp' postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local 'command=DROP DATABASE myapp' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3My11wYdfk +++ mktemp ++ local LAST_ERR=/tmp/tmp.LMdzRgG9Pi ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.3My11wYdfk ++ cat /tmp/tmp.LMdzRgG9Pi ++ rm /tmp/tmp.3My11wYdfk /tmp/tmp.LMdzRgG9Pi ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.YcSoNYpwKo ++ mktemp + local LAST_ERR=/tmp/tmp.tPEwfHGISD + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.YcSoNYpwKo + cat /tmp/tmp.tPEwfHGISD + rm /tmp/tmp.YcSoNYpwKo /tmp/tmp.tPEwfHGISD + return 0 + run_restore some-name-21191 incr s3 773 xid + local cluster=some-name-21191 + local rstr_prefix=incr + local storage=s3 + local target=773 + local type=xid + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/conf/restore.yml metadata.labels.pg-cluster some-name-21191 + yq w - metadata.name incr-some-name-21191 + yq w - spec.parameters.backrest-restore-cluster some-name-21191 + yq w - spec.parameters.backrest-storage-type s3 + yq w - spec.namespace scheduled-backup-6083 + yq w - spec.name incr-some-name-21191 + [[ 773 != \n\u\l\l ]] + yq w --style=single -- - spec.parameters.backrest-restore-opts --type=xid + yq w /tmp/tmp.yIkqhaYiTU/restore.yml --style=single spec.parameters.backrest-pitr-target 773 + mv /tmp/tmp.yIkqhaYiTU/restore.pitr.yml /tmp/tmp.yIkqhaYiTU/restore.yml + kubectl apply -f /tmp/tmp.yIkqhaYiTU/restore.yml pgtask.pg.percona.com/incr-some-name-21191 created + wait_bootstrap_completeness some-name-21191 + local cluster=some-name-21191 + wait_job_completion some-name-21191-bootstrap + local job=some-name-21191-bootstrap + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' some-name-21191-bootstrap+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_deployment some-name-21191-backrest-shared-repo + local name=some-name-21191-backrest-shared-repo + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-21191-backrest-shared-repo + wait_deployment some-name-21191 + local name=some-name-21191 + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-21191. + wait_job_completion some-name-21191-stanza-create false + local job=some-name-21191-stanza-create + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.metadata.name}' some-name-21191-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name-21191 + local job=backrest-backup-some-name-21191 + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-21191+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 48 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 49 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 50 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 51 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 52 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 53 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 54 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-21191 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo ++ get_psql_user_pass postgres some-name-21191 ++ local user=postgres ++ local cluster=some-name-21191 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.x9uhYPd1zT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tUjV9mqnqk +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.x9uhYPd1zT +++ cat /tmp/tmp.tUjV9mqnqk +++ rm /tmp/tmp.x9uhYPd1zT /tmp/tmp.tUjV9mqnqk +++ return 0 ++ [[ xsome-name-21191 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aoSTZXhiXq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6hPyd9ejqx +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-21191 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.aoSTZXhiXq +++ cat /tmp/tmp.6hPyd9ejqx +++ rm /tmp/tmp.aoSTZXhiXq /tmp/tmp.6hPyd9ejqx +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JVB1AmOE4G ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0BLZ5IsJWg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.JVB1AmOE4G +++ cat /tmp/tmp.0BLZ5IsJWg +++ rm /tmp/tmp.JVB1AmOE4G /tmp/tmp.0BLZ5IsJWg +++ return 0 ++ [[ xsome-name-21191-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.O0IRwjL9A6 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.NmuyJzfWDa ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-21191 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.O0IRwjL9A6 ++++ cat /tmp/tmp.NmuyJzfWDa ++++ rm /tmp/tmp.O0IRwjL9A6 /tmp/tmp.NmuyJzfWDa ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VMsRfZ63BW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xJPFrOK13o +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-21191-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.VMsRfZ63BW +++ cat /tmp/tmp.xJPFrOK13o +++ rm /tmp/tmp.VMsRfZ63BW /tmp/tmp.xJPFrOK13o +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-21191.scheduled-backup-6083 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-21191.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.brTdFfWqR5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.sfyNIZyZAu ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.brTdFfWqR5 ++ cat /tmp/tmp.sfyNIZyZAu ++ rm /tmp/tmp.brTdFfWqR5 /tmp/tmp.sfyNIZyZAu ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.mFjPj75DW6 ++ mktemp + local LAST_ERR=/tmp/tmp.NeZum2ttJS + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-21191.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.mFjPj75DW6 + cat /tmp/tmp.NeZum2ttJS + rm /tmp/tmp.mFjPj75DW6 /tmp/tmp.NeZum2ttJS + return 0 + '[' '!' -s /tmp/tmp.yIkqhaYiTU/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.yIkqhaYiTU/select-1.sql + delete_cluster_gracefully some-name-21191 + local cluster=some-name-21191 + kubectl_bin delete perconapgcluster/some-name-21191 ++ mktemp + local LAST_OUT=/tmp/tmp.1BNH7uEnRv ++ mktemp + local LAST_ERR=/tmp/tmp.iQ8yt3qCT8 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete perconapgcluster/some-name-21191 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.1BNH7uEnRv perconapgcluster.pg.percona.com "some-name-21191" deleted + cat /tmp/tmp.iQ8yt3qCT8 + rm /tmp/tmp.1BNH7uEnRv /tmp/tmp.iQ8yt3qCT8 + return 0 ++ kubectl_bin -n scheduled-backup-6083 get jobs --selector=pg-cluster=some-name-21191,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ssMUiBRHoz +++ mktemp ++ local LAST_ERR=/tmp/tmp.jHj9En83Bu ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get jobs --selector=pg-cluster=some-name-21191,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ssMUiBRHoz ++ cat /tmp/tmp.jHj9En83Bu ++ rm /tmp/tmp.ssMUiBRHoz /tmp/tmp.jHj9En83Bu ++ return 0 + rmjob_name=some-name-21191-rmdata-xfij + wait_job_completion some-name-21191-rmdata-xfij false + local job=some-name-21191-rmdata-xfij + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' some-name-21191-rmdata-xfij+ retry=0 ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 48 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 49 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 50 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 51 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 52 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 53 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 54 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 55 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-21191-rmdata-xfij -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + desc 'Testing full backup/restore from s3, local' + set +o xtrace ----------------------------------------------------------------------------------- Testing full backup/restore from s3, local ----------------------------------------------------------------------------------- + cluster=some-name-7396 + apply_configs some-name-7396 + local cluster=some-name-7396 + set +x secret/some-name-7396-backrest-repo-config created + yq r /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/conf/custom-config.yml + yq w - metadata.name some-name-7396-custom-config + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.0UxXOjmIsR ++ mktemp + local LAST_ERR=/tmp/tmp.L5uTQwAqyr + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.0UxXOjmIsR configmap/some-name-7396-custom-config created + cat /tmp/tmp.L5uTQwAqyr + rm /tmp/tmp.0UxXOjmIsR /tmp/tmp.L5uTQwAqyr + return 0 + spinup_pgcluster some-name-7396 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml local,s3 some-name-7396-custom-config false true full + local cluster=some-name-7396 + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local backup=local,s3 + local custom_config=some-name-7396-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=full + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + local namespace=scheduled-backup-6083 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name-7396 scheduled-backup-6083 + local cluster=some-name-7396 + local namespace=scheduled-backup-6083 + yq w - metadata.name some-name-7396-users-secret + yq w - stringData.some-name some-name_pass + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + kubectl_bin -n scheduled-backup-6083 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.x7E6LkpvQT ++ mktemp + local LAST_ERR=/tmp/tmp.6ibwFGKxyk + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.x7E6LkpvQT secret/some-name-7396-users-secret created + cat /tmp/tmp.6ibwFGKxyk + rm /tmp/tmp.x7E6LkpvQT /tmp/tmp.6ibwFGKxyk + return 0 + yq w - metadata.name some-name-7396-pmm-secret + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + kubectl_bin -n scheduled-backup-6083 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.f8cTYELNzp ++ mktemp + local LAST_ERR=/tmp/tmp.HD2RknZP9I + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.f8cTYELNzp secret/some-name-7396-pmm-secret created + cat /tmp/tmp.HD2RknZP9I + rm /tmp/tmp.f8cTYELNzp /tmp/tmp.HD2RknZP9I + return 0 + kubectl_bin -n scheduled-backup-6083 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.zmBw0rlwMz ++ mktemp + local LAST_ERR=/tmp/tmp.8kY1uwQKw2 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.zmBw0rlwMz deployment.apps/pg-client unchanged + cat /tmp/tmp.8kY1uwQKw2 + rm /tmp/tmp.zmBw0rlwMz /tmp/tmp.8kY1uwQKw2 + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name-7396 local,s3 false false some-name-7396-custom-config false true full false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local name=some-name-7396 + local backup=local,s3 + local restore_from=false + local restore_opts=false + local custom_config=some-name-7396-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=full + local tls_only=false + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name-7396 + yq w - metadata.labels.crunchy-pgha-scope some-name-7396 + yq w - spec.name some-name-7396 + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.pmm.image perconalab/pmm-client:2.41.0 + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.database some-name-7396 + yq w - spec.clustername some-name-7396 + yq w - spec.secretsName some-name-7396-users-secret + yq w - spec.user some-name + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.keepBackups false + yq w - metadata.name some-name-7396 + yq w - metadata.labels.pg-cluster some-name-7396 + yq w - spec.keepData false + yq w - metadata.labels.deployment-name some-name-7396 + yq w - metadata.labels.name some-name-7396 + case ${podAntiAffinity} in + case ${backup} in + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w /tmp/tmp.yIkqhaYiTU/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + [[ true == \t\r\u\e ]] + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.schedule[0].name' sch-backup + yq w - 'spec.backup.schedule[0].schedule' '*/5 * * * *' + yq w - 'spec.backup.schedule[0].type' full + yq w /tmp/tmp.yIkqhaYiTU/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.schedule[0].storage' my-s3 + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + mv /tmp/tmp.yIkqhaYiTU/cr.backup.yaml /tmp/tmp.yIkqhaYiTU/cr.yaml + [[ false != \f\a\l\s\e ]] + [[ some-name-7396-custom-config != \f\a\l\s\e ]] + yq w -i /tmp/tmp.yIkqhaYiTU/cr.yaml spec.pgPrimary.customconfig some-name-7396-custom-config + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.yIkqhaYiTU/cr.yaml -n scheduled-backup-6083 ++ mktemp + local LAST_OUT=/tmp/tmp.csfqRhBjxx ++ mktemp + local LAST_ERR=/tmp/tmp.T57e9MIT72 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f /tmp/tmp.yIkqhaYiTU/cr.yaml -n scheduled-backup-6083 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.csfqRhBjxx perconapgcluster.pg.percona.com/some-name-7396 created + cat /tmp/tmp.T57e9MIT72 + rm /tmp/tmp.csfqRhBjxx /tmp/tmp.T57e9MIT72 + return 0 + wait_deployment some-name-7396-backrest-shared-repo + local name=some-name-7396-backrest-shared-repo + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-7396-backrest-shared-repo. + wait_deployment some-name-7396 + local name=some-name-7396 + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-7396...... + [[ local,s3 == \g\c\s\+* ]] + wait_job_completion some-name-7396-stanza-create false + local job=some-name-7396-stanza-create + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' some-name-7396-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_job_completion backrest-backup-some-name-7396 + local job=backrest-backup-some-name-7396 + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-7396+ retry=0 ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name-7396 + cluster_name=some-name-7396 + wait_cluster_status some-name-7396 'pgcluster Initialized' + local cluster_name=some-name-7396 + local 'target_status=pgcluster Initialized' + local target_namespace=scheduled-backup-6083 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BTauUsdZQA +++ mktemp ++ local LAST_ERR=/tmp/tmp.LS0hk1yUjd ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.BTauUsdZQA ++ cat /tmp/tmp.LS0hk1yUjd ++ rm /tmp/tmp.BTauUsdZQA /tmp/tmp.LS0hk1yUjd ++ return 0 + status_query='kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fCQFjFyBAw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pdo1FNreDy +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.fCQFjFyBAw +++ cat /tmp/tmp.pdo1FNreDy +++ rm /tmp/tmp.fCQFjFyBAw /tmp/tmp.pdo1FNreDy +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name-7396 ++ local user=postgres ++ local cluster=some-name-7396 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zjJPOdI8zq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3eFzhLIHmH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zjJPOdI8zq +++ cat /tmp/tmp.3eFzhLIHmH +++ rm /tmp/tmp.zjJPOdI8zq /tmp/tmp.3eFzhLIHmH +++ return 0 ++ [[ xsome-name-7396 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RohaYJxmKv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wDEQDgJ3zD +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RohaYJxmKv +++ cat /tmp/tmp.wDEQDgJ3zD +++ rm /tmp/tmp.RohaYJxmKv /tmp/tmp.wDEQDgJ3zD +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iN9FbaoecU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AD2jVPAhLF +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.iN9FbaoecU +++ cat /tmp/tmp.AD2jVPAhLF +++ rm /tmp/tmp.iN9FbaoecU /tmp/tmp.AD2jVPAhLF +++ return 0 ++ [[ xsome-name-7396-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.BvqWe1hVKK +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.6IqxMYXV7c ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.BvqWe1hVKK ++++ cat /tmp/tmp.6IqxMYXV7c ++++ rm /tmp/tmp.BvqWe1hVKK /tmp/tmp.6IqxMYXV7c ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dfaUpuF8ia ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eUeRGo1tiK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.dfaUpuF8ia +++ cat /tmp/tmp.eUeRGo1tiK +++ rm /tmp/tmp.dfaUpuF8ia /tmp/tmp.eUeRGo1tiK +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.35LK2AgfiX +++ mktemp ++ local LAST_ERR=/tmp/tmp.jJrw8dttvB ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.35LK2AgfiX ++ cat /tmp/tmp.jJrw8dttvB ++ rm /tmp/tmp.35LK2AgfiX /tmp/tmp.jJrw8dttvB ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.6zkb64kJbd ++ mktemp + local LAST_ERR=/tmp/tmp.HGIr59hvZJ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.6zkb64kJbd + cat /tmp/tmp.HGIr59hvZJ + rm /tmp/tmp.6zkb64kJbd /tmp/tmp.HGIr59hvZJ + return 0 ++ get_psql_user_pass postgres some-name-7396 ++ local user=postgres ++ local cluster=some-name-7396 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VuePk2q0Ec ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VC7txoSY76 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.VuePk2q0Ec +++ cat /tmp/tmp.VC7txoSY76 +++ rm /tmp/tmp.VuePk2q0Ec /tmp/tmp.VC7txoSY76 +++ return 0 ++ [[ xsome-name-7396 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7cCuv7Gtpl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bAuF4cL9Tg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7cCuv7Gtpl +++ cat /tmp/tmp.bAuF4cL9Tg +++ rm /tmp/tmp.7cCuv7Gtpl /tmp/tmp.bAuF4cL9Tg +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OBDicpMHNu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bIaf8tkUb0 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.OBDicpMHNu +++ cat /tmp/tmp.bIaf8tkUb0 +++ rm /tmp/tmp.OBDicpMHNu /tmp/tmp.bIaf8tkUb0 +++ return 0 ++ [[ xsome-name-7396-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.nuNkGRxzes +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.X1qxi8Iiyu ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.nuNkGRxzes ++++ cat /tmp/tmp.X1qxi8Iiyu ++++ rm /tmp/tmp.nuNkGRxzes /tmp/tmp.X1qxi8Iiyu ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cJGSncZx6o ++++ mktemp +++ local LAST_ERR=/tmp/tmp.s13fdsCzv5 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.cJGSncZx6o +++ cat /tmp/tmp.s13fdsCzv5 +++ rm /tmp/tmp.cJGSncZx6o /tmp/tmp.s13fdsCzv5 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EFZC5vLSIs +++ mktemp ++ local LAST_ERR=/tmp/tmp.fBl46uCEh3 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.EFZC5vLSIs ++ cat /tmp/tmp.fBl46uCEh3 ++ rm /tmp/tmp.EFZC5vLSIs /tmp/tmp.fBl46uCEh3 ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.r90hlEScag ++ mktemp + local LAST_ERR=/tmp/tmp.kcDksGv4z6 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.r90hlEScag + cat /tmp/tmp.kcDksGv4z6 + rm /tmp/tmp.r90hlEScag /tmp/tmp.kcDksGv4z6 + return 0 ++ get_psql_user_pass postgres some-name-7396 ++ local user=postgres ++ local cluster=some-name-7396 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EForHt1URk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hhRB3pcsVg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.EForHt1URk +++ cat /tmp/tmp.hhRB3pcsVg +++ rm /tmp/tmp.EForHt1URk /tmp/tmp.hhRB3pcsVg +++ return 0 ++ [[ xsome-name-7396 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7GNa3TmzVA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SnJxtjWdcs +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7GNa3TmzVA +++ cat /tmp/tmp.SnJxtjWdcs +++ rm /tmp/tmp.7GNa3TmzVA /tmp/tmp.SnJxtjWdcs +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CUcBWntaVg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gZnMx33xyu +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.CUcBWntaVg +++ cat /tmp/tmp.gZnMx33xyu +++ rm /tmp/tmp.CUcBWntaVg /tmp/tmp.gZnMx33xyu +++ return 0 ++ [[ xsome-name-7396-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.xoHhCCCElP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.axUq4Dbxz7 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.xoHhCCCElP ++++ cat /tmp/tmp.axUq4Dbxz7 ++++ rm /tmp/tmp.xoHhCCCElP /tmp/tmp.axUq4Dbxz7 ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.flHmL2PUcO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ObCqqMrA5L +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.flHmL2PUcO +++ cat /tmp/tmp.ObCqqMrA5L +++ rm /tmp/tmp.flHmL2PUcO /tmp/tmp.ObCqqMrA5L +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fgOPobEQTF +++ mktemp ++ local LAST_ERR=/tmp/tmp.Auzyn4W0pR ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.fgOPobEQTF ++ cat /tmp/tmp.Auzyn4W0pR ++ rm /tmp/tmp.fgOPobEQTF /tmp/tmp.Auzyn4W0pR ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.cgLUmLUUzN ++ mktemp + local LAST_ERR=/tmp/tmp.jLgNl59aN2 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.cgLUmLUUzN + cat /tmp/tmp.jLgNl59aN2 + rm /tmp/tmp.cgLUmLUUzN /tmp/tmp.jLgNl59aN2 + return 0 + sleep 10 +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.backup.schedule[0].name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.e27HmRrGff ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6my0otbtIM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.backup.schedule[0].name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.e27HmRrGff +++ cat /tmp/tmp.6my0otbtIM +++ rm /tmp/tmp.e27HmRrGff /tmp/tmp.6my0otbtIM +++ return 0 ++ next_backup_after_sec some-name-7396-sch-backup ++ local conf_map_name=some-name-7396-sch-backup ++++ kubectl_bin get configmap/some-name-7396-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.HMCxBzPpSS +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.JxXGWUvKoh ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl get configmap/some-name-7396-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.HMCxBzPpSS ++++ cat /tmp/tmp.JxXGWUvKoh ++++ rm /tmp/tmp.HMCxBzPpSS /tmp/tmp.JxXGWUvKoh ++++ return 0 +++ /usr/bin/date +%s --date=2024-04-23T14:22:41Z --utc ++ local configmap_creation_time=1713882161 +++ kubectl_bin get configmap/some-name-7396-sch-backup -o 'jsonpath={.data.schedule}' +++ jq .schedule +++ grep -Eo '[0-9]+' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7usJUtrj8n ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jxrSoMjUkG +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get configmap/some-name-7396-sch-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7usJUtrj8n +++ cat /tmp/tmp.jxrSoMjUkG +++ rm /tmp/tmp.7usJUtrj8n /tmp/tmp.jxrSoMjUkG +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1713882161 300 +++ local start_time_sec=1713882161 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1713882461 +++ local next_interval_min_edge=161 +++ local next_backup_at_sec=1713882300 +++ echo 1713882300 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Tue, 23 Apr 2024 14:26:08 GMT ' ++++ /usr/bin/date --date 'Tue, 23 Apr 2024 14:26:08 GMT ' --utc +%s +++ echo 1713882368 ++ local result=-68 ++ [[ -68 -gt 0 ]] ++++ get_current_epoch_time +++++ curl -sI https://percona.com/ +++++ grep date +++++ /usr/bin/sed 's/date: //g' ++++ local 'curent_date=Tue, 23 Apr 2024 14:26:09 GMT ' +++++ /usr/bin/date --date 'Tue, 23 Apr 2024 14:26:09 GMT ' --utc +%s ++++ echo 1713882369 +++ calculate_next_backup_start 1713882369 300 +++ local start_time_sec=1713882369 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1713882669 +++ local next_interval_min_edge=69 +++ local next_backup_at_sec=1713882600 +++ echo 1713882600 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Tue, 23 Apr 2024 14:26:09 GMT ' ++++ /usr/bin/date --date 'Tue, 23 Apr 2024 14:26:09 GMT ' --utc +%s +++ echo 1713882369 ++ result=231 ++ [[ 231 -gt 0 ]] ++ echo 231 + sleep 231 + kubectl_bin wait job/some-name-7396-full-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.HU8QX4bcZX ++ mktemp + local LAST_ERR=/tmp/tmp.PKPOEhdOnB + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl wait job/some-name-7396-full-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.HU8QX4bcZX job.batch/some-name-7396-full-sch-backup condition met + cat /tmp/tmp.PKPOEhdOnB + rm /tmp/tmp.HU8QX4bcZX /tmp/tmp.PKPOEhdOnB + return 0 + restore some-name-7396 s3 full + local cluster=some-name-7396 + local storage=s3 + local backup_type=full + local restore_target= ++ get_psql_user_pass postgres some-name-7396 ++ local user=postgres ++ local cluster=some-name-7396 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KI4DNDPG4g ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kUEe3mpN2x +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.KI4DNDPG4g +++ cat /tmp/tmp.kUEe3mpN2x +++ rm /tmp/tmp.KI4DNDPG4g /tmp/tmp.kUEe3mpN2x +++ return 0 ++ [[ xsome-name-7396 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UFi8Tu7EJ7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HDWWQFMeLT +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.UFi8Tu7EJ7 +++ cat /tmp/tmp.HDWWQFMeLT +++ rm /tmp/tmp.UFi8Tu7EJ7 /tmp/tmp.HDWWQFMeLT +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.txzH8h587q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DTAroPOFbt +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.txzH8h587q +++ cat /tmp/tmp.DTAroPOFbt +++ rm /tmp/tmp.txzH8h587q /tmp/tmp.DTAroPOFbt +++ return 0 ++ [[ xsome-name-7396-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.CM6HYKuNvn +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.gwrTiVGFf6 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.CM6HYKuNvn ++++ cat /tmp/tmp.gwrTiVGFf6 ++++ rm /tmp/tmp.CM6HYKuNvn /tmp/tmp.gwrTiVGFf6 ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5MtUZWLMCg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5nnafTfgAK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.5MtUZWLMCg +++ cat /tmp/tmp.5nnafTfgAK +++ rm /tmp/tmp.5MtUZWLMCg /tmp/tmp.5nnafTfgAK +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EjpAD5RYU0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2MkzDk0GcG ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.EjpAD5RYU0 ++ cat /tmp/tmp.2MkzDk0GcG ++ rm /tmp/tmp.EjpAD5RYU0 /tmp/tmp.2MkzDk0GcG ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.DPiAFBvGWv ++ mktemp + local LAST_ERR=/tmp/tmp.mWpaZVBHDv + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.DPiAFBvGWv + cat /tmp/tmp.mWpaZVBHDv + rm /tmp/tmp.DPiAFBvGWv /tmp/tmp.mWpaZVBHDv + return 0 ++ get_psql_user_pass postgres some-name-7396 ++ local user=postgres ++ local cluster=some-name-7396 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PP5V3TZQYw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jPZEByOW4q +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.PP5V3TZQYw +++ cat /tmp/tmp.jPZEByOW4q +++ rm /tmp/tmp.PP5V3TZQYw /tmp/tmp.jPZEByOW4q +++ return 0 ++ [[ xsome-name-7396 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TicMAJ5vDy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UgwgdsSs4C +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TicMAJ5vDy +++ cat /tmp/tmp.UgwgdsSs4C +++ rm /tmp/tmp.TicMAJ5vDy /tmp/tmp.UgwgdsSs4C +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cqi6FGFaQU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rQF9kCw1yN +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.cqi6FGFaQU +++ cat /tmp/tmp.rQF9kCw1yN +++ rm /tmp/tmp.cqi6FGFaQU /tmp/tmp.rQF9kCw1yN +++ return 0 ++ [[ xsome-name-7396-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.A33NkMIJKG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.odLhhHtThW ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.A33NkMIJKG ++++ cat /tmp/tmp.odLhhHtThW ++++ rm /tmp/tmp.A33NkMIJKG /tmp/tmp.odLhhHtThW ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PCKUXigkw8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AVpUDIn9gT +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.PCKUXigkw8 +++ cat /tmp/tmp.AVpUDIn9gT +++ rm /tmp/tmp.PCKUXigkw8 /tmp/tmp.AVpUDIn9gT +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-7396.scheduled-backup-6083 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QRNPjq13TY +++ mktemp ++ local LAST_ERR=/tmp/tmp.CbZC1GzILN ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.QRNPjq13TY ++ cat /tmp/tmp.CbZC1GzILN ++ rm /tmp/tmp.QRNPjq13TY /tmp/tmp.CbZC1GzILN ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.btGaRzbUHl ++ mktemp + local LAST_ERR=/tmp/tmp.2HFq58bc5M + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.btGaRzbUHl + cat /tmp/tmp.2HFq58bc5M + rm /tmp/tmp.btGaRzbUHl /tmp/tmp.2HFq58bc5M + return 0 + '[' '!' -s /tmp/tmp.yIkqhaYiTU/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.yIkqhaYiTU/select-1.sql ++ get_psql_user_pass postgres some-name-7396 ++ local user=postgres ++ local cluster=some-name-7396 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Rwq1fnsscC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OByyXsAiRB +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Rwq1fnsscC +++ cat /tmp/tmp.OByyXsAiRB +++ rm /tmp/tmp.Rwq1fnsscC /tmp/tmp.OByyXsAiRB +++ return 0 ++ [[ xsome-name-7396 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CUpxX1tp0Y ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YGQYxm1DhJ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.CUpxX1tp0Y +++ cat /tmp/tmp.YGQYxm1DhJ +++ rm /tmp/tmp.CUpxX1tp0Y /tmp/tmp.YGQYxm1DhJ +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vt1BKeRgMo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VmtMjNJq3q +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.vt1BKeRgMo +++ cat /tmp/tmp.VmtMjNJq3q +++ rm /tmp/tmp.vt1BKeRgMo /tmp/tmp.VmtMjNJq3q +++ return 0 ++ [[ xsome-name-7396-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Abhp0Qte4v +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.QiWeLUZaTe ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.Abhp0Qte4v ++++ cat /tmp/tmp.QiWeLUZaTe ++++ rm /tmp/tmp.Abhp0Qte4v /tmp/tmp.QiWeLUZaTe ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sPtGjr0JOw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LIXRdysXmg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.sPtGjr0JOw +++ cat /tmp/tmp.LIXRdysXmg +++ rm /tmp/tmp.sPtGjr0JOw /tmp/tmp.LIXRdysXmg +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yHAiwUok19 +++ mktemp ++ local LAST_ERR=/tmp/tmp.29jDcc6hTh ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.yHAiwUok19 ++ cat /tmp/tmp.29jDcc6hTh ++ rm /tmp/tmp.yHAiwUok19 /tmp/tmp.29jDcc6hTh ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.qXypV4GOIb ++ mktemp + local LAST_ERR=/tmp/tmp.9zbrsIZreA + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.qXypV4GOIb + cat /tmp/tmp.9zbrsIZreA + rm /tmp/tmp.qXypV4GOIb /tmp/tmp.9zbrsIZreA + return 0 ++ awk '{$1=$1;print}' +++ get_psql_user_pass postgres some-name-7396 +++ local user=postgres +++ local cluster=some-name-7396 +++ local namespace=scheduled-backup-6083 ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.x5V8KToDHQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.XRcEhXQWh8 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.x5V8KToDHQ ++++ cat /tmp/tmp.XRcEhXQWh8 ++++ rm /tmp/tmp.x5V8KToDHQ /tmp/tmp.XRcEhXQWh8 ++++ return 0 +++ [[ xsome-name-7396 == \x ]] +++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ym9IT1X3rx +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.IL2Fp8UOly ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ym9IT1X3rx ++++ cat /tmp/tmp.IL2Fp8UOly ++++ rm /tmp/tmp.ym9IT1X3rx /tmp/tmp.IL2Fp8UOly ++++ return 0 ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.EPXbvbszmY +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.NQTQz8WUV0 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.EPXbvbszmY ++++ cat /tmp/tmp.NQTQz8WUV0 ++++ rm /tmp/tmp.EPXbvbszmY /tmp/tmp.NQTQz8WUV0 ++++ return 0 +++ [[ xsome-name-7396-users-secret == \x ]] ++++ base64 -d +++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WFFvWEAx5p ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.CKf8XYh5ey +++++ local exit_status=0 +++++ local timeout=4 +++++ set +o errexit ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ set -o errexit +++++ cat /tmp/tmp.WFFvWEAx5p +++++ cat /tmp/tmp.CKf8XYh5ey +++++ rm /tmp/tmp.WFFvWEAx5p /tmp/tmp.CKf8XYh5ey +++++ return 0 ++++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.OQ5tY2VQ4a +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.jPN2UU44rL ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.OQ5tY2VQ4a ++++ cat /tmp/tmp.jPN2UU44rL ++++ rm /tmp/tmp.OQ5tY2VQ4a /tmp/tmp.jPN2UU44rL ++++ return 0 +++ echo postgres_password ++ run_psql '\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' postgres:postgres_password@some-name-7396.scheduled-backup-6083 ++ local 'command=\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' ++ local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 ++ local driver=postgres ++ local suffix=.svc.cluster.local +++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FLF5iO1Xya ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pBua5BgeRT +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.FLF5iO1Xya +++ cat /tmp/tmp.pBua5BgeRT +++ rm /tmp/tmp.FLF5iO1Xya /tmp/tmp.pBua5BgeRT +++ return 0 ++ local client_container=pg-client-76df9df876-4grks ++ kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HsMARPiJej +++ mktemp ++ local LAST_ERR=/tmp/tmp.WPMWVLhpKE ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.HsMARPiJej ++ cat /tmp/tmp.WPMWVLhpKE ++ rm /tmp/tmp.HsMARPiJej /tmp/tmp.WPMWVLhpKE ++ return 0 + restore_target=773 ++ get_psql_user_pass postgres some-name-7396 ++ local user=postgres ++ local cluster=some-name-7396 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.otatxZ5ZfX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.QUeAAFzC6A +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.otatxZ5ZfX +++ cat /tmp/tmp.QUeAAFzC6A +++ rm /tmp/tmp.otatxZ5ZfX /tmp/tmp.QUeAAFzC6A +++ return 0 ++ [[ xsome-name-7396 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.t49SF4nf6w ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SiLRsIYsYb +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.t49SF4nf6w +++ cat /tmp/tmp.SiLRsIYsYb +++ rm /tmp/tmp.t49SF4nf6w /tmp/tmp.SiLRsIYsYb +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.w7ehwKQME9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CAIr3CDAXg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.w7ehwKQME9 +++ cat /tmp/tmp.CAIr3CDAXg +++ rm /tmp/tmp.w7ehwKQME9 /tmp/tmp.CAIr3CDAXg +++ return 0 ++ [[ xsome-name-7396-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.FxnFDMzmUf +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.5BdLoqA4Bd ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.FxnFDMzmUf ++++ cat /tmp/tmp.5BdLoqA4Bd ++++ rm /tmp/tmp.FxnFDMzmUf /tmp/tmp.5BdLoqA4Bd ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RbRv3juaJc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iLU7tdyerh +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RbRv3juaJc +++ cat /tmp/tmp.iLU7tdyerh +++ rm /tmp/tmp.RbRv3juaJc /tmp/tmp.iLU7tdyerh +++ return 0 ++ echo postgres_password + run_psql 'DROP DATABASE myapp' postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local 'command=DROP DATABASE myapp' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JO6doNiIeG +++ mktemp ++ local LAST_ERR=/tmp/tmp.2uxonO2hKT ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.JO6doNiIeG ++ cat /tmp/tmp.2uxonO2hKT ++ rm /tmp/tmp.JO6doNiIeG /tmp/tmp.2uxonO2hKT ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.YAQUwHJGDb ++ mktemp + local LAST_ERR=/tmp/tmp.4XUhS6RbSN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.YAQUwHJGDb + cat /tmp/tmp.4XUhS6RbSN + rm /tmp/tmp.YAQUwHJGDb /tmp/tmp.4XUhS6RbSN + return 0 + run_restore some-name-7396 full s3 773 xid + local cluster=some-name-7396 + local rstr_prefix=full + local storage=s3 + local target=773 + local type=xid + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/conf/restore.yml metadata.labels.pg-cluster some-name-7396 + yq w - metadata.name full-some-name-7396 + yq w - spec.parameters.backrest-restore-cluster some-name-7396 + yq w - spec.parameters.backrest-storage-type s3 + yq w - spec.namespace scheduled-backup-6083 + yq w - spec.name full-some-name-7396 + [[ 773 != \n\u\l\l ]] + yq w /tmp/tmp.yIkqhaYiTU/restore.yml --style=single spec.parameters.backrest-pitr-target 773 + yq w --style=single -- - spec.parameters.backrest-restore-opts --type=xid + mv /tmp/tmp.yIkqhaYiTU/restore.pitr.yml /tmp/tmp.yIkqhaYiTU/restore.yml + kubectl apply -f /tmp/tmp.yIkqhaYiTU/restore.yml pgtask.pg.percona.com/full-some-name-7396 created + wait_bootstrap_completeness some-name-7396 + local cluster=some-name-7396 + wait_job_completion some-name-7396-bootstrap + local job=some-name-7396-bootstrap + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' some-name-7396-bootstrap+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_deployment some-name-7396-backrest-shared-repo + local name=some-name-7396-backrest-shared-repo + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-7396-backrest-shared-repo + wait_deployment some-name-7396 + local name=some-name-7396 + local target_namespace=scheduled-backup-6083 + sleep 10 + set +o xtrace some-name-7396 + wait_job_completion some-name-7396-stanza-create false + local job=some-name-7396-stanza-create + local mandatory=false + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' some-name-7396-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/some-name-7396-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_job_completion backrest-backup-some-name-7396 + local job=backrest-backup-some-name-7396 + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-7396+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/backrest-backup-some-name-7396 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo ++ get_psql_user_pass postgres some-name-7396 ++ local user=postgres ++ local cluster=some-name-7396 ++ local namespace=scheduled-backup-6083 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2B64Sidz1k ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gVhLFY2mOA +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2B64Sidz1k +++ cat /tmp/tmp.gVhLFY2mOA +++ rm /tmp/tmp.2B64Sidz1k /tmp/tmp.gVhLFY2mOA +++ return 0 ++ [[ xsome-name-7396 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.E7eXKalQnZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.O7i4FJnaye +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-7396 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.E7eXKalQnZ +++ cat /tmp/tmp.O7i4FJnaye +++ rm /tmp/tmp.E7eXKalQnZ /tmp/tmp.O7i4FJnaye +++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pBP241k6lP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vdJb1gFYWB +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.pBP241k6lP +++ cat /tmp/tmp.vdJb1gFYWB +++ rm /tmp/tmp.pBP241k6lP /tmp/tmp.vdJb1gFYWB +++ return 0 ++ [[ xsome-name-7396-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ti7tY9ssdv +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PIDWXgACSm ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++++ kubectl -n scheduled-backup-6083 get perconapgcluster/some-name-7396 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ti7tY9ssdv ++++ cat /tmp/tmp.PIDWXgACSm ++++ rm /tmp/tmp.ti7tY9ssdv /tmp/tmp.PIDWXgACSm ++++ return 0 +++ kubectl_bin -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9AwEHmOpCK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Kp8ghSHKsP +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup +++ kubectl -n scheduled-backup-6083 get secret/some-name-7396-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.9AwEHmOpCK +++ cat /tmp/tmp.Kp8ghSHKsP +++ rm /tmp/tmp.9AwEHmOpCK /tmp/tmp.Kp8ghSHKsP +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-7396.scheduled-backup-6083 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-7396.scheduled-backup-6083 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8Cw0SmRYFM +++ mktemp ++ local LAST_ERR=/tmp/tmp.aQSh0fKpfG ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl -n scheduled-backup-6083 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.8Cw0SmRYFM ++ cat /tmp/tmp.aQSh0fKpfG ++ rm /tmp/tmp.8Cw0SmRYFM /tmp/tmp.aQSh0fKpfG ++ return 0 + local client_container=pg-client-76df9df876-4grks + kubectl_bin exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.hTrG5wdBFL ++ mktemp + local LAST_ERR=/tmp/tmp.IKGRe4ZIyf + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl exec pg-client-76df9df876-4grks -n scheduled-backup-6083 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-7396.scheduled-backup-6083.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.hTrG5wdBFL + cat /tmp/tmp.IKGRe4ZIyf + rm /tmp/tmp.hTrG5wdBFL /tmp/tmp.IKGRe4ZIyf + return 0 + '[' '!' -s /tmp/tmp.yIkqhaYiTU/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.yIkqhaYiTU/select-1.sql + destroy scheduled-backup-6083 ++ get_operator_pod + grep -v level=debug ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' + grep -v 'Getting tasks for pod' + grep -v 'Getting pods from source' + grep -v 'get backup status: Job.batch' + sort -u + tee /tmp/tmp.yIkqhaYiTU/operator.log +++ mktemp + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'the object has been modified' + grep -v level=info ++ local LAST_OUT=/tmp/tmp.Kctz3hmolG +++ mktemp ++ local LAST_ERR=/tmp/tmp.GPqdBVsf7m ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Kctz3hmolG ++ cat /tmp/tmp.GPqdBVsf7m ++ rm /tmp/tmp.Kctz3hmolG /tmp/tmp.GPqdBVsf7m ++ return 0 + kubectl_bin logs postgres-operator-78b79fcc87-bjpf4 -c operator ++ mktemp + local LAST_OUT=/tmp/tmp.mZweTh5QhK ++ mktemp + local LAST_ERR=/tmp/tmp.QbF4n2kEq9 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl logs postgres-operator-78b79fcc87-bjpf4 -c operator + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.mZweTh5QhK + cat /tmp/tmp.QbF4n2kEq9 + rm /tmp/tmp.mZweTh5QhK /tmp/tmp.QbF4n2kEq9 + return 0 ] { } }, ] } }, { } }, ] { } }, { } ] { } }, { } }, ] } ] { } }, }, { ] { } }, ] ] ], { } }, }, { }, { },{ }] }], ] { } }, , ], { } }, }, { }], { } }, }], } ], } }, } }, } }, { } }, },{ }, 2024/04/23 13:36:11 INF 1 (localhost:4150) connecting to nsqd 2024/04/23 13:37:32 INF 2 (localhost:4150) connecting to nsqd 2024/04/23 13:37:32 INF 3 (localhost:4150) connecting to nsqd 2024/04/23 13:37:55 INF 4 (localhost:4150) connecting to nsqd 2024/04/23 13:39:53 INF 5 (localhost:4150) connecting to nsqd 2024/04/23 13:39:54 INF 6 (localhost:4150) connecting to nsqd 2024/04/23 13:39:54 INF 7 (localhost:4150) connecting to nsqd 2024/04/23 13:40:00 INF 8 (localhost:4150) connecting to nsqd 2024/04/23 13:42:18 INF 9 (localhost:4150) connecting to nsqd 2024/04/23 13:45:00 INF 10 (localhost:4150) connecting to nsqd 2024/04/23 13:47:07 INF 11 (localhost:4150) connecting to nsqd 2024/04/23 13:47:49 INF 12 (localhost:4150) connecting to nsqd 2024/04/23 13:48:30 INF 13 (localhost:4150) connecting to nsqd 2024/04/23 13:49:08 INF 14 (localhost:4150) connecting to nsqd 2024/04/23 13:49:17 INF 15 (localhost:4150) connecting to nsqd 2024/04/23 13:50:00 INF 16 (localhost:4150) connecting to nsqd 2024/04/23 13:51:22 INF 17 (localhost:4150) connecting to nsqd 2024/04/23 13:51:23 INF 18 (localhost:4150) connecting to nsqd 2024/04/23 13:51:23 INF 19 (localhost:4150) connecting to nsqd 2024/04/23 13:51:31 INF 20 (localhost:4150) connecting to nsqd 2024/04/23 13:52:23 INF 21 (localhost:4150) connecting to nsqd 2024/04/23 13:52:28 INF 22 (localhost:4150) connecting to nsqd 2024/04/23 13:52:48 INF 23 (localhost:4150) connecting to nsqd 2024/04/23 13:53:37 INF 24 (localhost:4150) connecting to nsqd 2024/04/23 13:53:38 INF 25 (localhost:4150) connecting to nsqd 2024/04/23 13:53:47 INF 26 (localhost:4150) connecting to nsqd 2024/04/23 13:55:00 INF 27 (localhost:4150) connecting to nsqd 2024/04/23 13:55:44 INF 28 (localhost:4150) connecting to nsqd 2024/04/23 13:55:44 INF 29 (localhost:4150) connecting to nsqd 2024/04/23 13:55:45 INF 30 (localhost:4150) connecting to nsqd 2024/04/23 13:56:01 INF 31 (localhost:4150) connecting to nsqd 2024/04/23 14:00:00 INF 32 (localhost:4150) connecting to nsqd 2024/04/23 14:00:45 INF 33 (localhost:4150) connecting to nsqd 2024/04/23 14:01:26 INF 34 (localhost:4150) connecting to nsqd 2024/04/23 14:02:02 INF 35 (localhost:4150) connecting to nsqd 2024/04/23 14:03:01 INF 36 (localhost:4150) connecting to nsqd 2024/04/23 14:03:09 INF 37 (localhost:4150) connecting to nsqd 2024/04/23 14:05:00 INF 38 (localhost:4150) connecting to nsqd 2024/04/23 14:05:18 INF 39 (localhost:4150) connecting to nsqd 2024/04/23 14:05:18 INF 40 (localhost:4150) connecting to nsqd 2024/04/23 14:05:18 INF 41 (localhost:4150) connecting to nsqd 2024/04/23 14:05:26 INF 42 (localhost:4150) connecting to nsqd 2024/04/23 14:07:43 INF 43 (localhost:4150) connecting to nsqd 2024/04/23 14:07:48 INF 44 (localhost:4150) connecting to nsqd 2024/04/23 14:08:09 INF 45 (localhost:4150) connecting to nsqd 2024/04/23 14:08:58 INF 46 (localhost:4150) connecting to nsqd 2024/04/23 14:08:59 INF 47 (localhost:4150) connecting to nsqd 2024/04/23 14:09:08 INF 48 (localhost:4150) connecting to nsqd 2024/04/23 14:10:00 INF 49 (localhost:4150) connecting to nsqd 2024/04/23 14:11:10 INF 50 (localhost:4150) connecting to nsqd 2024/04/23 14:11:10 INF 51 (localhost:4150) connecting to nsqd 2024/04/23 14:11:10 INF 52 (localhost:4150) connecting to nsqd 2024/04/23 14:12:08 INF 53 (localhost:4150) connecting to nsqd 2024/04/23 14:15:00 INF 54 (localhost:4150) connecting to nsqd 2024/04/23 14:15:21 INF 55 (localhost:4150) connecting to nsqd 2024/04/23 14:16:05 INF 56 (localhost:4150) connecting to nsqd 2024/04/23 14:16:41 INF 57 (localhost:4150) connecting to nsqd 2024/04/23 14:17:23 INF 58 (localhost:4150) connecting to nsqd 2024/04/23 14:17:32 INF 59 (localhost:4150) connecting to nsqd 2024/04/23 14:19:41 INF 60 (localhost:4150) connecting to nsqd 2024/04/23 14:19:41 INF 61 (localhost:4150) connecting to nsqd 2024/04/23 14:19:42 INF 62 (localhost:4150) connecting to nsqd 2024/04/23 14:19:48 INF 63 (localhost:4150) connecting to nsqd 2024/04/23 14:22:05 INF 64 (localhost:4150) connecting to nsqd 2024/04/23 14:22:10 INF 65 (localhost:4150) connecting to nsqd 2024/04/23 14:22:32 INF 66 (localhost:4150) connecting to nsqd 2024/04/23 14:23:19 INF 67 (localhost:4150) connecting to nsqd 2024/04/23 14:23:20 INF 68 (localhost:4150) connecting to nsqd 2024/04/23 14:23:28 INF 69 (localhost:4150) connecting to nsqd 2024/04/23 14:25:00 INF 70 (localhost:4150) connecting to nsqd 2024/04/23 14:25:26 INF 71 (localhost:4150) connecting to nsqd 2024/04/23 14:25:26 INF 72 (localhost:4150) connecting to nsqd 2024/04/23 14:25:27 INF 73 (localhost:4150) connecting to nsqd 2024/04/23 14:27:51 INF 74 (localhost:4150) connecting to nsqd 2024/04/23 14:30:00 INF 75 (localhost:4150) connecting to nsqd 2024/04/23 14:32:04 INF 76 (localhost:4150) connecting to nsqd 2024/04/23 14:32:46 INF 77 (localhost:4150) connecting to nsqd 2024/04/23 14:33:29 INF 78 (localhost:4150) connecting to nsqd 2024/04/23 14:34:21 INF 79 (localhost:4150) connecting to nsqd 2024/04/23 14:34:29 INF 80 (localhost:4150) connecting to nsqd 2024/04/23 14:35:00 INF 81 (localhost:4150) connecting to nsqd 2024/04/23 14:36:38 INF 82 (localhost:4150) connecting to nsqd 2024/04/23 14:36:39 INF 83 (localhost:4150) connecting to nsqd 2024/04/23 14:36:39 INF 84 (localhost:4150) connecting to nsqd "accessModes": [ "affinity": "affinity": { "affinity": { "podAntiAffinity": { "allowPrivilegeEscalation": false, "annotations": {"keep-backups":"false","keep-data":"false"}, "apiVersion": "apps/v1", "apiVersion": "batch/v1", "apiVersion": "v1", "apiVersion": "v1", "apiVersion": "v1", "backrest-command": "backup", "backrest-command": "stanza-create", "claimName": "some-name-21191-pgbr-repo" "claimName": "some-name-4738-pgbr-repo" "claimName": "some-name-5931-pgbr-repo" "claimName": "some-name-7396-pgbr-repo" "command": [ "configMap": { "configMap": { "configMap": { "configMapKeyRef": { "containerName": "database", "containerPort": 2022, "containerPort": 5432, "containerPort": 8009, "containers": [ "containers": [{ "cpu": "1", "cpu": "200m", "cpu": "500m", "crunchydata" "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "crunchy-pgha-scope": "some-name-21191","pgo-version": "1.6.0","name": "some-name-21191","pg-cluster": "some-name-21191","deployment-name": "some-name-21191","pgouser": "admin" "crunchy-pgha-scope": "some-name-7396","pgo-version": "1.6.0","name": "some-name-7396","pg-cluster": "some-name-7396","deployment-name": "some-name-7396","pgouser": "admin" "defaultMode": 288 "defaultMode": 288, "defaultMode": 288, "defaultMode": 420, "deployment-name": "some-name-21191" "deployment-name": "some-name-21191","pgouser": "admin","crunchy-pgha-scope": "some-name-21191","pgo-version": "1.6.0","name": "some-name-21191","pg-cluster": "some-name-21191" "deployment-name": "some-name-21191","pgouser": "admin","crunchy-pgha-scope": "some-name-21191","pgo-version": "1.6.0","name": "some-name-21191","pg-cluster": "some-name-21191" "deployment-name": "some-name-21191-repl1" "deployment-name": "some-name-21191-repl2" "deployment-name": "some-name-4738" "deployment-name": "some-name-4738-repl1" "deployment-name": "some-name-4738-repl2" "deployment-name": "some-name-5931" "deployment-name": "some-name-5931","pgouser": "admin","crunchy-pgha-scope": "some-name-5931","pgo-version": "1.6.0","name": "some-name-5931","pg-cluster": "some-name-5931" "deployment-name": "some-name-5931","pgouser": "admin","crunchy-pgha-scope": "some-name-5931","pgo-version": "1.6.0","name": "some-name-5931","pg-cluster": "some-name-5931" "deployment-name": "some-name-5931-repl1" "deployment-name": "some-name-5931-repl1","pgo-version": "1.6.0","name": "some-name-5931-replica","pg-cluster": "some-name-5931" "deployment-name": "some-name-5931-repl2" "deployment-name": "some-name-7396" "deployment-name": "some-name-7396-repl1" "deployment-name": "some-name-7396-repl2" "divisor": "1m", "dnsPolicy": "ClusterFirst" "downwardAPI": { "emptyDir": { "emptyDir": { "emptyDir": { "emptyDir": { "env": [ "env": [{ "exec": { "fieldPath": "metadata.annotations" "fieldPath": "metadata.labels" "fieldPath": "metadata.labels['crunchy-pgha-scope']" "fieldPath": "metadata.namespace" "fieldRef": { "fieldRef": { "fsGroup": 2, I0423 13:35:28.289912 1 shared_informer.go:240] Waiting for caches to sync for namespace I0423 13:35:28.390379 1 shared_informer.go:247] Caches are synced for namespace I0423 13:35:28.521596 1 shared_informer.go:240] Waiting for caches to sync for scheduled-backup-6083 I0423 13:35:28.621908 1 shared_informer.go:247] Caches are synced for scheduled-backup-6083 "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer", "image": "perconalab/percona-postgresql-operator:main-ppg14-postgres-ha", "image": "perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-rmdata", "imagePullPolicy": "IfNotPresent" "imagePullPolicy": "IfNotPresent", "initialDelaySeconds": 15 "initialDelaySeconds": 30, "items": [ "items": [ "items": [ "items": [ "key": "aws-s3-key" "key": "aws-s3-key-secret" "key": "config", "key": "config", "key": "config", "key": "crunchy-pgbouncer", "key": "init" "key": "password" "key": "pg-cluster", "key": "pgo-backrest", "key": "pgo-backrest-job", "key": "pgo-pg-database", "key": "pg-pod-anti-affinity", "key": "tls.crt", "key": "tls.key", "key": "vendor", "kind": "Deployment", "kind": "Job", "kind": "PersistentVolumeClaim", "kind": "Service", "labels": { "labels": { "labels": { "labelSelector": { "livenessProbe": { "matchExpressions": [ "matchLabels": { "matchLabels": { "maxSurge": 1 "maxUnavailable": 1, "medium": "Memory", "medium": "Memory" "medium": "Memory", "medium": "Memory", "medium": "Memory", "memory": "128Mi" "memory": "256Mi" "memory": "48Mi" "metadata": { "metadata": { "metadata": { "mountPath": "/backrestrepo", "mountPath": "/dev/shm", "mountPath": "/dev/shm", "mountPath": "/etc/pgbackrest" "mountPath": "/etc/pgbackrest/conf.d", "mountPath": "/etc/pgbackrest/conf.d", "mountPath": "/etc/podinfo", "mountPath": "/etc/ssh", "mountPath": "/etc/ssh", "mountPath": "/home/pgbackrest" "mountPath": "/pgconf", "mountPath": "/pgconf/", "mountPath": "/pgconf", "mountPath": "/pgconf/pgreplicator", "mountPath": "/pgconf/pgreplicator", "mountPath": "/pgconf/pgsuper", "mountPath": "/pgconf/pgsuper", "mountPath": "/pgconf/pguser", "mountPath": "/pgconf/pguser", "mountPath": "/pgconf/tls", "mountPath": "/pgconf/tls", "mountPath": "/pgconf/tls/pgbouncer", "mountPath": "/pgconf/tls-replication", "mountPath": "/pgdata", "mountPath": "/pgdata", "mountPath": "/sshd", "mountPath": "/sshd", "mountPath": "/sshd", "mountPath": "/tmp", "mountPath": "/tmp", "mountPath": "/tmp", "mountPath": "/tmp" "name": "backrest", "name": "backrest-backup-some-name-21191", "name": "backrest-backup-some-name-4738", "name": "backrest-backup-some-name-5931", "name": "backrest-backup-some-name-7396", "name": "backrestrepo", "name": "backrestrepo", "name": "BACKREST_SKIP_CREATE_STANZA", "name": "cluster1-replication-ssl-keypair", "name": "cluster1-ssl-ca" "name": "cluster1-ssl-ca" "name": "cluster1-ssl-ca" "name": "cluster1-ssl-keypair" "name": "cluster1-ssl-keypair" "name": "cluster1-ssl-keypair" "name": "COMMAND", "name": "COMMAND_OPTS", "name": "database", "name": "dshm" "name": "dshm" "name": "dshm", "name": "dshm", "name": "ENABLE_SSHD", "name": "IS_BACKUP", "name": "IS_REPLICA", "name": "LD_PRELOAD", "name": "LD_PRELOAD", "name": "MODE", "name": "NAMESPACE", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_PASSWD", "name": "NSS_WRAPPER_PASSWD", "name": "patroni" "name": "PATRONI_KUBERNETES_LABELS", "name": "PATRONI_KUBERNETES_NAMESPACE", "name": "PATRONI_KUBERNETES_SCOPE_LABEL", "name": "PATRONI_LOG_LEVEL", "name": "PATRONI_POSTGRESQL_DATA_DIR", "name": "PATRONI_SCOPE", "name": "pgbackrest" "name": "pgbackrest-conf", "name": "pgbackrest-conf", "name": "pgbackrest-config" "name": "pgbackrest-config" "name": "pgbackrest-config", "name": "pgbackrest-config", "name": "PGBACKREST_DB_HOST", "name": "PGBACKREST_DB_PATH", "name": "PGBACKREST_DB_PATH", "name": "pgbackrest-home", "name": "pgbackrest-home", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_REPO1_HOST", "name": "PGBACKREST_REPO1_HOST_CMD", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_S3_BUCKET", "name": "PGBACKREST_REPO1_S3_CA_FILE", "name": "PGBACKREST_REPO1_S3_ENDPOINT", "name": "PGBACKREST_REPO1_S3_KEY", "name": "PGBACKREST_REPO1_S3_KEY_SECRET", "name": "PGBACKREST_REPO1_S3_REGION", "name": "PGBACKREST_REPO1_S3_URI_STYLE", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_STANZA", "name": "PGBACKREST_STANZA", "name": "pgbouncer" "name": "pgbouncer", "name": "pgbouncer-conf", "name": "pgbouncer-conf", "name": "PG_CLUSTER", "name": "pgconf-volume" "name": "pgconf-volume" "name": "pgconf-volume", "name": "pgconf-volume", "name": "pgdata", "name": "pgdata" "name": "pgdata", "name": "pgdata", "name": "PGHA_BOOTSTRAP_METHOD", "name": "PGHA_DATABASE", "name": "PGHA_INIT", "name": "PGHA_PASSWORD_TYPE", "name": "PGHA_PGBACKREST", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_S3_VERIFY_TLS", "name": "PGHA_PGBACKREST_S3_VERIFY_TLS", "name": "PGHA_PG_PORT", "name": "PGHA_REPLICA_REINIT_ON_START_FAIL", "name": "PGHA_SCOPE", "name": "PGHA_STANDBY", "name": "PGHA_SYNC_REPLICATION", "name": "PGHA_TLS_ENABLED", "name": "PGHA_TLS_ONLY", "name": "PGHA_USER", "name": "PGHOST", "name": "PG_PASSWORD", "name": "PG_PRIMARY_SERVICE_NAME", "name": "PITR_TARGET", "name": "podinfo" "name": "podinfo", "name": "PODNAME", "name": "postgres" "name": "primary-volume" "name": "primary-volume" "name": "primary-volume", "name": "primary-volume", "name": "REMOVE_BACKUP", "name": "REMOVE_DATA", "name": "REPLICA_NAME", "name": "report", "name": "RESTORE_OPTS", "name": "rmdata", "name": "root-volume" "name": "root-volume" "name": "root-volume", "name": "root-volume", "name": "some-name-21191", "name": "some-name-21191", "name": "some-name-21191-backrest-repo-config", "name": "some-name-21191-backrest-shared-repo", "name": "some-name-21191-backrest-shared-repo" "name": "some-name-21191-backrest-shared-repo", "name": "some-name-21191-backrest-shared-repo", "name": "some-name-21191-backrest-shared-repo", "name": "some-name-21191-bootstrap", "name": "some-name-21191-bootstrap", "name": "some-name-21191-custom-config" "name": "some-name-21191-custom-config" "name": "some-name-21191-incr-sch-backup", "name": "some-name-21191-pgbouncer", "name": "some-name-21191-pgbouncer", "name": "some-name-21191-pgbouncer", "name": "some-name-21191-pgbouncer", "name": "some-name-21191-pgbouncer-cm" "name": "some-name-21191-pgbouncer-secret", "name": "some-name-21191-pgbouncer-secret", "name": "some-name-21191-pgbr-repo", "name": "some-name-21191-pgha-config", "name": "some-name-21191-pgha-config", "name": "some-name-21191-pgha-config", "name": "some-name-21191-repl1", "name": "some-name-21191-repl1", "name": "some-name-21191-repl2", "name": "some-name-21191-repl2", "name": "some-name-21191-replica","pg-cluster": "some-name-21191","deployment-name": "some-name-21191-repl2","pgo-version": "1.6.0" "name": "some-name-21191-rmdata-xfij", "name": "some-name-21191-rmdata-xfij", "name": "some-name-21191-stanza-create", "name": "some-name-4738", "name": "some-name-4738", "name": "some-name-4738-backrest-repo-config", "name": "some-name-4738-backrest-shared-repo", "name": "some-name-4738-backrest-shared-repo" "name": "some-name-4738-backrest-shared-repo", "name": "some-name-4738-backrest-shared-repo", "name": "some-name-4738-backrest-shared-repo", "name": "some-name-4738-bootstrap", "name": "some-name-4738-bootstrap", "name": "some-name-4738-custom-config" "name": "some-name-4738-custom-config" "name": "some-name-4738-diff-sch-backup", "name": "some-name-4738-pgbouncer", "name": "some-name-4738-pgbouncer", "name": "some-name-4738-pgbouncer", "name": "some-name-4738-pgbouncer", "name": "some-name-4738-pgbouncer-cm" "name": "some-name-4738-pgbouncer-secret", "name": "some-name-4738-pgbouncer-secret", "name": "some-name-4738-pgbr-repo", "name": "some-name-4738","pg-cluster": "some-name-4738","deployment-name": "some-name-4738","pgouser": "admin","crunchy-pgha-scope": "some-name-4738","pgo-version": "1.6.0" "name": "some-name-4738-pgha-config", "name": "some-name-4738-pgha-config", "name": "some-name-4738-pgha-config", "name": "some-name-4738-repl1", "name": "some-name-4738-repl1", "name": "some-name-4738-repl2", "name": "some-name-4738-repl2", "name": "some-name-4738-replica","pg-cluster": "some-name-4738","deployment-name": "some-name-4738-repl1","pgo-version": "1.6.0" "name": "some-name-4738-replica","pg-cluster": "some-name-4738","deployment-name": "some-name-4738-repl2","pgo-version": "1.6.0" "name": "some-name-4738-rmdata-nesv", "name": "some-name-4738-rmdata-nesv", "name": "some-name-4738-stanza-create", "name": "some-name-5931", "name": "some-name-5931", "name": "some-name-5931-backrest-repo-config", "name": "some-name-5931-backrest-shared-repo", "name": "some-name-5931-backrest-shared-repo" "name": "some-name-5931-backrest-shared-repo", "name": "some-name-5931-backrest-shared-repo", "name": "some-name-5931-backrest-shared-repo", "name": "some-name-5931-bootstrap", "name": "some-name-5931-bootstrap", "name": "some-name-5931-custom-config" "name": "some-name-5931-custom-config" "name": "some-name-5931-full-sch-backup", "name": "some-name-5931-pgbouncer", "name": "some-name-5931-pgbouncer", "name": "some-name-5931-pgbouncer", "name": "some-name-5931-pgbouncer", "name": "some-name-5931-pgbouncer-cm" "name": "some-name-5931-pgbouncer-secret", "name": "some-name-5931-pgbouncer-secret", "name": "some-name-5931-pgbr-repo", "name": "some-name-5931","pg-cluster": "some-name-5931","deployment-name": "some-name-5931","pgouser": "admin","crunchy-pgha-scope": "some-name-5931","pgo-version": "1.6.0" "name": "some-name-5931-pgha-config", "name": "some-name-5931-pgha-config", "name": "some-name-5931-pgha-config", "name": "some-name-5931-repl1", "name": "some-name-5931-repl1", "name": "some-name-5931-repl2", "name": "some-name-5931-repl2", "name": "some-name-5931-replica","pg-cluster": "some-name-5931","deployment-name": "some-name-5931-repl1","pgo-version": "1.6.0" "name": "some-name-5931-replica","pg-cluster": "some-name-5931","deployment-name": "some-name-5931-repl2","pgo-version": "1.6.0" "name": "some-name-5931-rmdata-autu", "name": "some-name-5931-rmdata-autu", "name": "some-name-5931-stanza-create", "name": "some-name-7396", "name": "some-name-7396", "name": "some-name-7396-backrest-repo-config", "name": "some-name-7396-backrest-shared-repo", "name": "some-name-7396-backrest-shared-repo" "name": "some-name-7396-backrest-shared-repo", "name": "some-name-7396-backrest-shared-repo", "name": "some-name-7396-backrest-shared-repo", "name": "some-name-7396-bootstrap", "name": "some-name-7396-bootstrap", "name": "some-name-7396-custom-config" "name": "some-name-7396-custom-config" "name": "some-name-7396-full-sch-backup", "name": "some-name-7396-pgbouncer", "name": "some-name-7396-pgbouncer", "name": "some-name-7396-pgbouncer", "name": "some-name-7396-pgbouncer", "name": "some-name-7396-pgbouncer-cm" "name": "some-name-7396-pgbouncer-secret", "name": "some-name-7396-pgbouncer-secret", "name": "some-name-7396-pgbr-repo", "name": "some-name-7396","pg-cluster": "some-name-7396","deployment-name": "some-name-7396","pgouser": "admin","crunchy-pgha-scope": "some-name-7396","pgo-version": "1.6.0" "name": "some-name-7396-pgha-config", "name": "some-name-7396-pgha-config", "name": "some-name-7396-pgha-config", "name": "some-name-7396-repl1", "name": "some-name-7396-repl1", "name": "some-name-7396-repl2", "name": "some-name-7396-repl2", "name": "some-name-7396-replica","pg-cluster": "some-name-7396","deployment-name": "some-name-7396-repl1","pgo-version": "1.6.0" "name": "some-name-7396-replica","pg-cluster": "some-name-7396","deployment-name": "some-name-7396-repl1","pgo-version": "1.6.0" "name": "some-name-7396-replica","pg-cluster": "some-name-7396","deployment-name": "some-name-7396-repl2","pgo-version": "1.6.0" "name": "some-name-7396-stanza-create", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "sshd", "name": "sshd", "name": "sshd", "name": "sshd", "name": "tls-pgbouncer" "name": "tls-pgbouncer", "name": "tls-replication" "name": "tls-replication", "name": "tls-server" "name": "tls-server" "name": "tls-server", "name": "tls-server", "name": "tmp" "name": "tmp" "name": "tmp" "name": "tmp", "name": "tmp", "name": "tmp", "name": "user-volume" "name": "user-volume" "name": "user-volume", "name": "user-volume", "nodePort": 0 "operator": "Exists" "operator": "In", "/opt/crunchy/bin/postgres-ha/health/pgha-liveness.sh" "/opt/crunchy/bin/postgres-ha/health/pgha-readiness.sh" "optional": true "optional": true "path": "annotations" "path": "cpu_limit", "path": "cpu_request", "path": "labels" "path": "mem_limit", "path": "mem_request", "path": "ssh_config" "path": "ssh_config" "path": "ssh_config" "path": "tls-replication.crt" "path": "tls-replication.key" "periodSeconds": 15, "persistentVolumeClaim": { "persistentVolumeClaim":{"claimName":"some-name-21191"} "persistentVolumeClaim":{"claimName":"some-name-21191"} "persistentVolumeClaim":{"claimName":"some-name-21191-repl1"} "persistentVolumeClaim":{"claimName":"some-name-21191-repl2"} "persistentVolumeClaim":{"claimName":"some-name-4738"} "persistentVolumeClaim":{"claimName":"some-name-4738"} "persistentVolumeClaim":{"claimName":"some-name-4738-repl1"} "persistentVolumeClaim":{"claimName":"some-name-4738-repl2"} "persistentVolumeClaim":{"claimName":"some-name-5931"} "persistentVolumeClaim":{"claimName":"some-name-5931"} "persistentVolumeClaim":{"claimName":"some-name-5931-repl1"} "persistentVolumeClaim":{"claimName":"some-name-5931-repl2"} "persistentVolumeClaim":{"claimName":"some-name-7396"} "persistentVolumeClaim":{"claimName":"some-name-7396"} "persistentVolumeClaim":{"claimName":"some-name-7396-repl1"} "persistentVolumeClaim":{"claimName":"some-name-7396-repl2"} "pg-cluster": "some-name-21191" "pg-cluster": "some-name-21191", "pg-cluster": "some-name-21191" "pg-cluster": "some-name-21191", "pg-cluster": "some-name-21191" "pg-cluster": "some-name-21191", "pg-cluster": "some-name-21191","deployment-name": "some-name-21191-repl1","pgo-version": "1.6.0","name": "some-name-21191-replica" "pg-cluster": "some-name-21191","deployment-name": "some-name-21191-repl1","pgo-version": "1.6.0","name": "some-name-21191-replica" "pg-cluster": "some-name-4738" "pg-cluster": "some-name-4738", "pg-cluster": "some-name-4738" "pg-cluster": "some-name-4738", "pg-cluster": "some-name-4738" "pg-cluster": "some-name-4738", "pg-cluster": "some-name-4738","deployment-name": "some-name-4738","pgouser": "admin","crunchy-pgha-scope": "some-name-4738","pgo-version": "1.6.0","name": "some-name-4738" "pg-cluster": "some-name-4738","deployment-name": "some-name-4738-repl2","pgo-version": "1.6.0","name": "some-name-4738-replica" "pg-cluster": "some-name-5931" "pg-cluster": "some-name-5931", "pg-cluster": "some-name-5931" "pg-cluster": "some-name-5931", "pg-cluster": "some-name-5931" "pg-cluster": "some-name-5931", "pg-cluster": "some-name-5931","deployment-name": "some-name-5931-repl1","pgo-version": "1.6.0","name": "some-name-5931-replica" "pg-cluster": "some-name-7396" "pg-cluster": "some-name-7396", "pg-cluster": "some-name-7396" "pg-cluster": "some-name-7396", "pg-cluster": "some-name-7396" "pg-cluster": "some-name-7396", "pg-cluster": "some-name-7396","deployment-name": "some-name-7396","pgouser": "admin","crunchy-pgha-scope": "some-name-7396","pgo-version": "1.6.0","name": "some-name-7396" "pgha-bootstrap": "some-name-21191", "pgha-bootstrap": "some-name-21191", "pgha-bootstrap": "some-name-4738", "pgha-bootstrap": "some-name-4738", "pgha-bootstrap": "some-name-5931", "pgha-bootstrap": "some-name-5931", "pgha-bootstrap": "some-name-7396", "pgha-bootstrap": "some-name-7396", "pgo-backrest-job": "true", "pgo-backrest-job": "true", "pgo-backrest-repo": "true" "pgo-backrest-repo": "true", "pgo-backrest-repo": "true" "pgo-backrest": "true", "pgo-pg-database": "true", "pgo-pg-database": "true", "pgouser": "admin","crunchy-pgha-scope": "some-name-21191","pgo-version": "1.6.0","name": "some-name-21191","pg-cluster": "some-name-21191","deployment-name": "some-name-21191" "pgouser": "admin","crunchy-pgha-scope": "some-name-4738","pgo-version": "1.6.0","name": "some-name-4738","pg-cluster": "some-name-4738","deployment-name": "some-name-4738" "pgo-version": "1.6.0","name": "some-name-21191","pg-cluster": "some-name-21191","deployment-name": "some-name-21191","pgouser": "admin","crunchy-pgha-scope": "some-name-21191" "pgo-version": "1.6.0","name": "some-name-21191-replica","pg-cluster": "some-name-21191","deployment-name": "some-name-21191-repl1" "pgo-version": "1.6.0","name": "some-name-21191-replica","pg-cluster": "some-name-21191","deployment-name": "some-name-21191-repl1" "pgo-version": "1.6.0","name": "some-name-21191-replica","pg-cluster": "some-name-21191","deployment-name": "some-name-21191-repl2" "pgo-version": "1.6.0","name": "some-name-21191-replica","pg-cluster": "some-name-21191","deployment-name": "some-name-21191-repl2" "pgo-version": "1.6.0","name": "some-name-4738","pg-cluster": "some-name-4738","deployment-name": "some-name-4738","pgouser": "admin","crunchy-pgha-scope": "some-name-4738" "pgo-version": "1.6.0","name": "some-name-4738","pg-cluster": "some-name-4738","deployment-name": "some-name-4738","pgouser": "admin","crunchy-pgha-scope": "some-name-4738" "pgo-version": "1.6.0","name": "some-name-4738-replica","pg-cluster": "some-name-4738","deployment-name": "some-name-4738-repl1" "pgo-version": "1.6.0","name": "some-name-4738-replica","pg-cluster": "some-name-4738","deployment-name": "some-name-4738-repl1" "pgo-version": "1.6.0","name": "some-name-4738-replica","pg-cluster": "some-name-4738","deployment-name": "some-name-4738-repl2" "pgo-version": "1.6.0","name": "some-name-5931","pg-cluster": "some-name-5931","deployment-name": "some-name-5931","pgouser": "admin","crunchy-pgha-scope": "some-name-5931" "pgo-version": "1.6.0","name": "some-name-5931-replica","pg-cluster": "some-name-5931","deployment-name": "some-name-5931-repl1" "pgo-version": "1.6.0","name": "some-name-5931-replica","pg-cluster": "some-name-5931","deployment-name": "some-name-5931-repl2" "pgo-version": "1.6.0","name": "some-name-5931-replica","pg-cluster": "some-name-5931","deployment-name": "some-name-5931-repl2" "pgo-version": "1.6.0","name": "some-name-7396","pg-cluster": "some-name-7396","deployment-name": "some-name-7396","pgouser": "admin","crunchy-pgha-scope": "some-name-7396" "pgo-version": "1.6.0","name": "some-name-7396","pg-cluster": "some-name-7396","deployment-name": "some-name-7396","pgouser": "admin","crunchy-pgha-scope": "some-name-7396" "pgo-version": "1.6.0","name": "some-name-7396-replica","pg-cluster": "some-name-7396","deployment-name": "some-name-7396-repl1" "pgo-version": "1.6.0","name": "some-name-7396-replica","pg-cluster": "some-name-7396","deployment-name": "some-name-7396-repl1" "pgo-version": "1.6.0","name": "some-name-7396-replica","pg-cluster": "some-name-7396","deployment-name": "some-name-7396-repl2" "pgo-version": "1.6.0","name": "some-name-7396-replica","pg-cluster": "some-name-7396","deployment-name": "some-name-7396-repl2" "pg-pod-anti-affinity": "preferred", "pg-pod-anti-affinity": "preferred", "pgrmdata": "true", "pgrmdata": "true", "podAffinityTerm": { "podAntiAffinity": { "podAntiAffinity": { "podAntiAffinity": { "podAntiAffinity": { "port": 2022, "ports": [{ "ports": [{ "preferredDuringSchedulingIgnoredDuringExecution": [ "privileged": false, "projected": { "projected": { "projected": { "projected": { "projected": { "sources": [] } "projected": { "sources": [] } "protocol": "TCP", "protocol": "TCP", "readinessProbe": { "readOnly": false "readOnly": false "readOnly": false "readOnlyRootFilesystem": true "readOnly": true "readOnly": true "readOnly": true "readOnly": true "ReadWriteOnce" ReadWriteOnce" "replicas": 0, "replicas": 1, "replicas": 3, "requests": { "requests": { "resourceFieldRef": { "resource": "limits.cpu" "resource": "limits.memory" "resource": "requests.cpu" "resource": "requests.memory" "resources": { "resources": { "restartPolicy": "Always", "restartPolicy": "Never" "rollingUpdate": { "runAsNonRoot": true "secret": { "secret": { "secret": { "secret": { "secret": { "secretKeyRef": { "secretKeyRef": { "secretName": "some-name-21191-backrest-repo-config" "secretName": "some-name-21191-backrest-repo-config", "secretName": "some-name-21191-backrest-repo-config" "secretName": "some-name-21191-backrest-repo-config", "secretName": "some-name-21191-bootstrap-backrest-repo-config" "secretName": "some-name-21191-bootstrap-backrest-repo-config", "secretName": "some-name-21191-postgres-secret" "secretName": "some-name-21191-postgres-secret" "secretName": "some-name-21191-primaryuser-secret" "secretName": "some-name-21191-primaryuser-secret" "secretName": "some-name-21191-some-name-secret" "secretName": "some-name-21191-some-name-secret" "secretName": "some-name-4738-backrest-repo-config" "secretName": "some-name-4738-backrest-repo-config", "secretName": "some-name-4738-backrest-repo-config" "secretName": "some-name-4738-backrest-repo-config", "secretName": "some-name-4738-bootstrap-backrest-repo-config" "secretName": "some-name-4738-bootstrap-backrest-repo-config", "secretName": "some-name-4738-postgres-secret" "secretName": "some-name-4738-postgres-secret" "secretName": "some-name-4738-primaryuser-secret" "secretName": "some-name-4738-primaryuser-secret" "secretName": "some-name-4738-some-name-secret" "secretName": "some-name-4738-some-name-secret" "secretName": "some-name-5931-backrest-repo-config" "secretName": "some-name-5931-backrest-repo-config", "secretName": "some-name-5931-backrest-repo-config" "secretName": "some-name-5931-backrest-repo-config", "secretName": "some-name-5931-bootstrap-backrest-repo-config" "secretName": "some-name-5931-bootstrap-backrest-repo-config", "secretName": "some-name-5931-postgres-secret" "secretName": "some-name-5931-postgres-secret" "secretName": "some-name-5931-primaryuser-secret" "secretName": "some-name-5931-primaryuser-secret" "secretName": "some-name-5931-some-name-secret" "secretName": "some-name-5931-some-name-secret" "secretName": "some-name-7396-backrest-repo-config" "secretName": "some-name-7396-backrest-repo-config", "secretName": "some-name-7396-backrest-repo-config" "secretName": "some-name-7396-backrest-repo-config", "secretName": "some-name-7396-bootstrap-backrest-repo-config" "secretName": "some-name-7396-bootstrap-backrest-repo-config", "secretName": "some-name-7396-postgres-secret" "secretName": "some-name-7396-postgres-secret" "secretName": "some-name-7396-primaryuser-secret" "secretName": "some-name-7396-primaryuser-secret" "secretName": "some-name-7396-some-name-secret" "secretName": "some-name-7396-some-name-secret" "securityContext": { "securityContext": { "securityContext": {}, "securityContext": {"fsGroup": 26,"supplementalGroups": [1001]}, "securityContext": {"runAsNonRoot":true,"fsGroup":26}, "selector": { "selector": { "serviceAccountName": "pgo-backrest", "serviceAccountName": "pgo-default", "serviceAccountName": "pgo-pg", "serviceAccountName": "pgo-target", "service-name": "some-name-21191-backrest-shared-repo", "service-name": "some-name-21191-backrest-shared-repo", "service-name": "some-name-21191-pgbouncer", "service-name": "some-name-21191-pgbouncer", "service-name": "some-name-21191-pgbouncer", "service-name": "some-name-4738-backrest-shared-repo", "service-name": "some-name-4738-backrest-shared-repo", "service-name": "some-name-4738-pgbouncer", "service-name": "some-name-4738-pgbouncer", "service-name": "some-name-4738-pgbouncer", "service-name": "some-name-5931-backrest-shared-repo", "service-name": "some-name-5931-backrest-shared-repo", "service-name": "some-name-5931-pgbouncer", "service-name": "some-name-5931-pgbouncer", "service-name": "some-name-5931-pgbouncer", "service-name": "some-name-7396-backrest-shared-repo", "service-name": "some-name-7396-backrest-shared-repo", "service-name": "some-name-7396-pgbouncer", "service-name": "some-name-7396-pgbouncer", "service-name": "some-name-7396-pgbouncer", "sessionAffinity": "None" "sizeLimit": "128Ki" "sizeLimit": "16Mi" "sizeLimit": "16Mi" "sizeLimit": "1Mi" "sizeLimit": "2Mi" "sizeLimit": "64Mi" "sizeLimit": "64Mi" "some-name-21191" "some-name-4738" "some-name-5931" "some-name-7396" "sources": [ "sources": [ "sources": [ "sources": [ "spec": { "spec": { "spec": { "storage": "1Gi" "strategy": { "targetPort": 2022, "template": { time="2024-04-23T13:35:27Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.6.0 time="2024-04-23T13:36:15Z" level=error msg="get deployment some-name-5931-repl1: deployments.apps \"some-name-5931-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T13:36:20Z" level=error msg="get deployment some-name-5931-repl2: deployments.apps \"some-name-5931-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T13:37:32Z" level=error msg="Error completing workflow some-name-5931-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T13:37:32Z" level=error msg="pgtasks.pg.percona.com \"some-name-5931-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T13:39:54Z" level=warning msg="crv1 pgreplica some-name-5931-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T13:39:54Z" level=warning msg="crv1 pgreplica some-name-5931-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T13:47:49Z" level=error msg="pgtask Controller: restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pgtask.(*Controller).handleBackrestRestore()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pgtask/backresthandler.go:72" version=1.6.0 time="2024-04-23T13:47:49Z" level=error msg="restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/operator/backrest.UpdateWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/backrest/restore.go:301" version=1.6.0 time="2024-04-23T13:48:28Z" level=error msg="configmaps \"some-name-5931-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.6.0 time="2024-04-23T13:48:28Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-5931-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.6.0 time="2024-04-23T13:49:08Z" level=error msg="Error completing workflow some-name-5931-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T13:49:08Z" level=error msg="pgtasks.pg.percona.com \"some-name-5931-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T13:49:09Z" level=warning msg="deployments.apps \"some-name-5931-pgbouncer\" already exists" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).handleBootstrapInit()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/inithandler.go:187" version=1.6.0 time="2024-04-23T13:51:23Z" level=warning msg="crv1 pgreplica some-name-5931-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T13:51:23Z" level=warning msg="crv1 pgreplica some-name-5931-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T13:51:28Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.6.0 time="2024-04-23T13:51:28Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-23T13:51:28Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.6.0 time="2024-04-23T13:51:31Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:51:33Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:51:34Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:51:35Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:51:36Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:51:38Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:51:39Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:09Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:12Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:13Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:16Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:17Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:20Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:22Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:23Z" level=error msg="pgclusters.pg.percona.com \"some-name-5931\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T13:52:28Z" level=error msg="jobs.batch \"some-name-5931-rmdata-autu\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0 time="2024-04-23T13:52:52Z" level=error msg="get deployment some-name-4738-repl1: deployments.apps \"some-name-4738-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T13:52:57Z" level=error msg="get deployment some-name-4738-repl2: deployments.apps \"some-name-4738-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T13:53:37Z" level=error msg="Error completing workflow some-name-4738-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T13:53:37Z" level=error msg="pgtasks.pg.percona.com \"some-name-4738-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T13:55:45Z" level=warning msg="crv1 pgreplica some-name-4738-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T13:55:45Z" level=warning msg="crv1 pgreplica some-name-4738-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:01:27Z" level=error msg="pgtask Controller: restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pgtask.(*Controller).handleBackrestRestore()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pgtask/backresthandler.go:72" version=1.6.0 time="2024-04-23T14:01:27Z" level=error msg="restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/operator/backrest.UpdateWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/backrest/restore.go:301" version=1.6.0 time="2024-04-23T14:02:01Z" level=error msg="configmaps \"some-name-4738-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.6.0 time="2024-04-23T14:02:01Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-4738-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.6.0 time="2024-04-23T14:03:01Z" level=error msg="Error completing workflow some-name-4738-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T14:03:01Z" level=error msg="pgtasks.pg.percona.com \"some-name-4738-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T14:03:01Z" level=warning msg="deployments.apps \"some-name-4738-pgbouncer\" already exists" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).handleBootstrapInit()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/inithandler.go:187" version=1.6.0 time="2024-04-23T14:05:19Z" level=warning msg="crv1 pgreplica some-name-4738-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:05:19Z" level=warning msg="crv1 pgreplica some-name-4738-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:05:26Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:05:27Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:05:28Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-4738-pgha-config (namespace scheduled-backup-6083),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.6.0 time="2024-04-23T14:05:28Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:05:29Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:05:30Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:05:31Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:05:33Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:05:34Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:06:28Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-4738-pgha-config (namespace scheduled-backup-6083),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.6.0 time="2024-04-23T14:07:29Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:07:31Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:07:35Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:07:36Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:07:40Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:07:42Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:07:43Z" level=error msg="pgclusters.pg.percona.com \"some-name-4738\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:07:48Z" level=error msg="jobs.batch \"some-name-4738-rmdata-nesv\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0 time="2024-04-23T14:08:12Z" level=error msg="get deployment some-name-21191-repl1: deployments.apps \"some-name-21191-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T14:08:17Z" level=error msg="get deployment some-name-21191-repl2: deployments.apps \"some-name-21191-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T14:08:58Z" level=error msg="Error completing workflow some-name-21191-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T14:08:58Z" level=error msg="pgtasks.pg.percona.com \"some-name-21191-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T14:11:10Z" level=warning msg="crv1 pgreplica some-name-21191-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:11:11Z" level=warning msg="crv1 pgreplica some-name-21191-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:16:05Z" level=error msg="pgtask Controller: restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pgtask.(*Controller).handleBackrestRestore()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pgtask/backresthandler.go:72" version=1.6.0 time="2024-04-23T14:16:05Z" level=error msg="restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/operator/backrest.UpdateWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/backrest/restore.go:301" version=1.6.0 time="2024-04-23T14:16:40Z" level=error msg="configmaps \"some-name-21191-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.6.0 time="2024-04-23T14:16:40Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-21191-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.6.0 time="2024-04-23T14:17:23Z" level=error msg="Error completing workflow some-name-21191-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T14:17:23Z" level=error msg="pgtasks.pg.percona.com \"some-name-21191-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T14:17:24Z" level=warning msg="deployments.apps \"some-name-21191-pgbouncer\" already exists" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).handleBootstrapInit()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/inithandler.go:187" version=1.6.0 time="2024-04-23T14:19:42Z" level=warning msg="crv1 pgreplica some-name-21191-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:19:43Z" level=warning msg="crv1 pgreplica some-name-21191-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:19:43Z" level=warning msg="crv1 pgreplica some-name-21191-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:19:48Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:19:51Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:19:52Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:19:53Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:19:54Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:19:56Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:19:57Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:20:27Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:20:28Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-21191-pgha-config (namespace scheduled-backup-6083),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.6.0 time="2024-04-23T14:21:52Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:21:53Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:21:57Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:21:59Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:22:02Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:22:04Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:22:05Z" level=error msg="pgclusters.pg.percona.com \"some-name-21191\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-23T14:22:10Z" level=error msg="jobs.batch \"some-name-21191-rmdata-xfij\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0 time="2024-04-23T14:22:36Z" level=error msg="get deployment some-name-7396-repl1: deployments.apps \"some-name-7396-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T14:22:41Z" level=error msg="get deployment some-name-7396-repl2: deployments.apps \"some-name-7396-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T14:23:19Z" level=error msg="Error completing workflow some-name-7396-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T14:23:19Z" level=error msg="pgtasks.pg.percona.com \"some-name-7396-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T14:25:27Z" level=warning msg="crv1 pgreplica some-name-7396-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:25:27Z" level=warning msg="crv1 pgreplica some-name-7396-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:25:28Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0 time="2024-04-23T14:25:28Z" level=error msg="pod some-name-7396-repl1-cb78c6b86-vglb9 does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-23T14:25:28Z" level=error msg="pod some-name-7396-repl1-cb78c6b86-vglb9 does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0 time="2024-04-23T14:25:28Z" level=error msg="pod some-name-7396-repl2-6fc6cc89c4-knj2s does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-23T14:25:28Z" level=error msg="pod some-name-7396-repl2-6fc6cc89c4-knj2s does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0 time="2024-04-23T14:32:46Z" level=error msg="pgtask Controller: restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pgtask.(*Controller).handleBackrestRestore()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pgtask/backresthandler.go:72" version=1.6.0 time="2024-04-23T14:32:46Z" level=error msg="restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/operator/backrest.UpdateWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/backrest/restore.go:301" version=1.6.0 time="2024-04-23T14:33:28Z" level=error msg="configmaps \"some-name-7396-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.6.0 time="2024-04-23T14:33:28Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-7396-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.6.0 time="2024-04-23T14:34:21Z" level=error msg="Error completing workflow some-name-7396-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T14:34:21Z" level=error msg="pgtasks.pg.percona.com \"some-name-7396-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T14:34:21Z" level=warning msg="deployments.apps \"some-name-7396-pgbouncer\" already exists" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).handleBootstrapInit()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/inithandler.go:187" version=1.6.0 time="2024-04-23T14:36:39Z" level=warning msg="crv1 pgreplica some-name-7396-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T14:36:39Z" level=warning msg="crv1 pgreplica some-name-7396-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 "timeoutSeconds": 10 "topologyKey": "kubernetes.io/hostname" "type": "ClusterIP", "type": "RollingUpdate", "value": "" "value": "5432" "value": "5432" "value": "/backrestrepo/some-name-21191-backrest-shared-repo" "value": "/backrestrepo/some-name-21191-backrest-shared-repo" "value": "/backrestrepo/some-name-4738-backrest-shared-repo" "value": "/backrestrepo/some-name-4738-backrest-shared-repo" "value": "/backrestrepo/some-name-5931-backrest-shared-repo" "value": "/backrestrepo/some-name-5931-backrest-shared-repo" "value": "/backrestrepo/some-name-7396-backrest-shared-repo" "value": "/backrestrepo/some-name-7396-backrest-shared-repo" "value": "backup" "value": "crunchy-pgha-scope" "value": "db" "value": "db" "value": " --db-host=10.178.72.10 --db-path=/pgdata/some-name-5931" "value": " --db-host=10.178.72.12 --db-path=/pgdata/some-name-4738" "value": " --db-host=10.178.72.17 --db-path=/pgdata/some-name-21191" "value": " --db-host=10.178.72.22 --db-path=/pgdata/some-name-21191" "value": " --db-host=10.178.72.24 --db-path=/pgdata/some-name-7396" "value": " --db-host=10.178.72.5 --db-path=/pgdata/some-name-5931" "value": " --db-host=10.178.73.14 --db-path=/pgdata/some-name-4738" "value": " --db-host=10.178.73.23 --db-path=/pgdata/some-name-7396" "value": "false" "value": "false" "valueFrom": { "valueFrom": { "value": "INFO" "value": "path" "value": "pgbackrest" "value": "pgbackrest_init" "value": "pgbackrest-repo" "value": "/pgdata/some-name-21191" "value": "/pgdata/some-name-21191" "value": "/pgdata/some-name-21191-repl1" "value": "/pgdata/some-name-21191-repl1" "value": "/pgdata/some-name-21191-repl2" "value": "/pgdata/some-name-21191-repl2" "value": "/pgdata/some-name-4738" "value": "/pgdata/some-name-4738" "value": "/pgdata/some-name-4738-repl1" "value": "/pgdata/some-name-4738-repl1" "value": "/pgdata/some-name-4738-repl2" "value": "/pgdata/some-name-4738-repl2" "value": "/pgdata/some-name-5931" "value": "/pgdata/some-name-5931" "value": "/pgdata/some-name-5931-repl1" "value": "/pgdata/some-name-5931-repl1" "value": "/pgdata/some-name-5931-repl2" "value": "/pgdata/some-name-5931-repl2" "value": "/pgdata/some-name-7396" "value": "/pgdata/some-name-7396" "value": "/pgdata/some-name-7396-repl1" "value": "/pgdata/some-name-7396-repl1" "value": "/pgdata/some-name-7396-repl2" "value": "/pgdata/some-name-7396-repl2" "value": "pg-operator-testing" "value": "posix" "value": "posix" "value": "postgres" "values": [ "value": "s3.amazonaws.com" "value": "some-name-21191" "value": "some-name-21191-backrest-shared-repo" "value": "some-name-21191-backrest-shared-repo-7d99596c8b-2mzxx" "value": "some-name-21191-backrest-shared-repo-7d99596c8b-fw5mv" "value": "some-name-21191-backrest-shared-repo.scheduled-backup-6083.svc.cluster.local." "value": "some-name-4738" "value": "some-name-4738-backrest-shared-repo" "value": "some-name-4738-backrest-shared-repo-d9bf5df64-8bj4r" "value": "some-name-4738-backrest-shared-repo-d9bf5df64-k9ngk" "value": "some-name-4738-backrest-shared-repo.scheduled-backup-6083.svc.cluster.local." "value": "some-name-5931" "value": "some-name-5931-backrest-shared-repo" "value": "some-name-5931-backrest-shared-repo-754775d78d-9kl42" "value": "some-name-5931-backrest-shared-repo-754775d78d-wbpr2" "value": "some-name-5931-backrest-shared-repo.scheduled-backup-6083.svc.cluster.local." "value": "some-name-7396" "value": "some-name-7396-backrest-shared-repo" "value": "some-name-7396-backrest-shared-repo-7cc66574c4-q2v7h" "value": "some-name-7396-backrest-shared-repo-7cc66574c4-rt8qm" "value": "some-name-7396-backrest-shared-repo.scheduled-backup-6083.svc.cluster.local." "value": "/sshd/aws-s3-ca.crt" "value": "stanza-create" "value": "--stanza=db --type=diff --repo1-retention-diff=3 --db-host=10.178.72.12 --db-path=/pgdata/some-name-4738" "value": "--stanza=db --type=diff --repo1-retention-diff=3 --db-host=10.178.73.14 --db-path=/pgdata/some-name-4738" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.178.72.10 --db-path=/pgdata/some-name-5931" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.178.72.24 --db-path=/pgdata/some-name-7396" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.178.72.5 --db-path=/pgdata/some-name-5931" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.178.73.23 --db-path=/pgdata/some-name-7396" "value": "--stanza=db --type=incr --db-host=10.178.72.17 --db-path=/pgdata/some-name-21191" "value": "/tmp" "value": "/tmp" "value": "/tmp/nss_wrapper/pgbackrest-repo/group" "value": "/tmp/nss_wrapper/pgbackrest-repo/passwd" "value": "/tmp/nss_wrapper/postgres/group" "value": "/tmp/nss_wrapper/postgres/passwd" "value": "true" "value": "true" "value": "--type=full --db-host=10.178.72.10 --db-path=/pgdata/some-name-5931" "value": "--type=full --db-host=10.178.72.12 --db-path=/pgdata/some-name-4738" "value": "--type=full --db-host=10.178.72.17 --db-path=/pgdata/some-name-21191" "value": "--type=full --db-host=10.178.72.22 --db-path=/pgdata/some-name-21191" "value": "--type=full --db-host=10.178.72.24 --db-path=/pgdata/some-name-7396" "value": "--type=full --db-host=10.178.72.5 --db-path=/pgdata/some-name-5931" "value": "--type=full --db-host=10.178.73.14 --db-path=/pgdata/some-name-4738" "value": "--type=full --db-host=10.178.73.23 --db-path=/pgdata/some-name-7396" "value": "--type=xid --target=\"773\" --repo-type=s3 --no-repo1-s3-verify-tls --target-action=promote" "value": "us-east-1" "value": "/usr/lib64/libnss_wrapper.so" "value": "/usr/lib64/libnss_wrapper.so" "value": "/usr/local/bin/archive-push-s3.sh" "value": "{vendor: \"crunchydata\"}" "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "volumeMounts": [ "volumeMounts": [{ "volumes": [ "volumes": [{ "weight": 1, + destroy_operator + kubectl_bin delete -f /tmp/tmp.yIkqhaYiTU/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.WwGc5m6AV7 ++ mktemp + local LAST_ERR=/tmp/tmp.4bQi0uzKRD + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete -f /tmp/tmp.yIkqhaYiTU/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete -f /tmp/tmp.yIkqhaYiTU/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete -f /tmp/tmp.yIkqhaYiTU/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.WwGc5m6AV7 + cat /tmp/tmp.4bQi0uzKRD Error from server (NotFound): error when deleting "/tmp/tmp.yIkqhaYiTU/operator.yaml": serviceaccounts "pgo-deployer-sa" not found Error from server (NotFound): error when deleting "/tmp/tmp.yIkqhaYiTU/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): error when deleting "/tmp/tmp.yIkqhaYiTU/operator.yaml": configmaps "pgo-deployer-cm" not found Error from server (NotFound): error when deleting "/tmp/tmp.yIkqhaYiTU/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found Error from server (NotFound): error when deleting "/tmp/tmp.yIkqhaYiTU/operator.yaml": jobs.batch "pgo-deploy" not found + rm /tmp/tmp.WwGc5m6AV7 /tmp/tmp.4bQi0uzKRD + return 1 + true + yq w -d4 /tmp/tmp.yIkqhaYiTU/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall + kubectl_bin apply -f /tmp/tmp.yIkqhaYiTU/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.btV8kprIuf ++ mktemp + local LAST_ERR=/tmp/tmp.Qb9r14WSd7 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl apply -f /tmp/tmp.yIkqhaYiTU/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.btV8kprIuf serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.Qb9r14WSd7 + rm /tmp/tmp.btV8kprIuf /tmp/tmp.Qb9r14WSd7 + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=scheduled-backup-6083 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-6083 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.yIkqhaYiTU/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fyi4XqDsIM ++ mktemp + local LAST_ERR=/tmp/tmp.dkhuNxpRqW + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete -f /tmp/tmp.yIkqhaYiTU/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.fyi4XqDsIM serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.dkhuNxpRqW + rm /tmp/tmp.fyi4XqDsIM /tmp/tmp.dkhuNxpRqW + return 0 + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + awk '{print$1}' ++ mktemp + local LAST_OUT=/tmp/tmp.QsjNxLN99M + xargs kubectl delete ns --grace-period=0 --force=true ++ mktemp + local LAST_ERR=/tmp/tmp.EMOEVTcFIU + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.QsjNxLN99M + cat /tmp/tmp.EMOEVTcFIU + rm /tmp/tmp.QsjNxLN99M /tmp/tmp.EMOEVTcFIU + return 0 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. namespace "scheduled-backup-6083" force deleted + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.I1yDg2SZU3 ++ mktemp + local LAST_ERR=/tmp/tmp.ouaDPfZHrM + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.I1yDg2SZU3 + cat /tmp/tmp.ouaDPfZHrM Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.I1yDg2SZU3 /tmp/tmp.ouaDPfZHrM + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.iYaMi609I5 ++ mktemp + local LAST_ERR=/tmp/tmp.kMMTMGnJdN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.iYaMi609I5 + cat /tmp/tmp.kMMTMGnJdN Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.iYaMi609I5 /tmp/tmp.kMMTMGnJdN + return 1 + true + rm -rf /tmp/tmp.yIkqhaYiTU