++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/self-healing.log' Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/self-healing.log ++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: the server doesn't have a resource type "projects" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' ']' +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.26 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.14.4 ++ '[' v3 == v2 ']' + main + create_namespace self-healing-10347 + local namespace=self-healing-10347 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + awk '{print$1}' + cleanup_rbac + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.o1Ns1LBYCm ++ mktemp + local LAST_OUT=/tmp/tmp.QvANsHxbnD ++ mktemp + local LAST_ERR=/tmp/tmp.FqjOfK2FxZ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + local LAST_ERR=/tmp/tmp.tQxJ3QSG5F + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.QvANsHxbnD + cat /tmp/tmp.tQxJ3QSG5F + rm /tmp/tmp.QvANsHxbnD /tmp/tmp.tQxJ3QSG5F + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.o1Ns1LBYCm + cat /tmp/tmp.FqjOfK2FxZ Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.o1Ns1LBYCm /tmp/tmp.FqjOfK2FxZ + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + local LAST_OUT=/tmp/tmp.qgnQrswuzv ++ mktemp + local LAST_ERR=/tmp/tmp.c7VNKVHiB0 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.qgnQrswuzv + cat /tmp/tmp.c7VNKVHiB0 Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.qgnQrswuzv /tmp/tmp.c7VNKVHiB0 + return 1 + true + kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found + true + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin delete namespace self-healing-10347 ++ mktemp + local LAST_OUT=/tmp/tmp.LenI4siYE2 ++ mktemp + local LAST_ERR=/tmp/tmp.k0vnn7kqvX + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete namespace self-healing-10347 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete namespace self-healing-10347 + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete namespace self-healing-10347 + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.LenI4siYE2 + cat /tmp/tmp.k0vnn7kqvX Error from server (NotFound): namespaces "self-healing-10347" not found + rm /tmp/tmp.LenI4siYE2 /tmp/tmp.k0vnn7kqvX + return 1 + : + wait_for_delete namespace/self-healing-10347 + local res=namespace/self-healing-10347 + set +o xtrace namespace/self-healing-10347 - Error from server (NotFound): namespaces "self-healing-10347" not found + kubectl_bin create namespace self-healing-10347 ++ mktemp + local LAST_OUT=/tmp/tmp.Y7fbVJrpqp ++ mktemp + local LAST_ERR=/tmp/tmp.Orf4GpzE67 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl create namespace self-healing-10347 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Y7fbVJrpqp namespace/self-healing-10347 created + cat /tmp/tmp.Orf4GpzE67 + rm /tmp/tmp.Y7fbVJrpqp /tmp/tmp.Orf4GpzE67 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.GuIpW01r6f +++ mktemp ++ local LAST_ERR=/tmp/tmp.L3jpiMB5iH ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.GuIpW01r6f ++ cat /tmp/tmp.L3jpiMB5iH ++ rm /tmp/tmp.GuIpW01r6f /tmp/tmp.L3jpiMB5iH ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox --namespace=self-healing-10347 ++ mktemp + local LAST_OUT=/tmp/tmp.DEoEoitsNL ++ mktemp + local LAST_ERR=/tmp/tmp.wQaAgSL3No + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox --namespace=self-healing-10347 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.DEoEoitsNL Context "gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox" modified. + cat /tmp/tmp.wQaAgSL3No + rm /tmp/tmp.DEoEoitsNL /tmp/tmp.wQaAgSL3No + return 0 + deploy_operator + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml + local namespace_mode=disabled + local action=install + local namespace=self-healing-10347 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml disabled install self-healing-10347 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml + local namespace_mode=disabled + local operator_action=install + local namespace=self-healing-10347 + local dont_send_telemetry=false + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml 'data[values.yaml]' + /usr/bin/sed -e 's#^namespace: .*#namespace: "self-healing-10347"#g' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "self-healing-10347"#g' + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g' + cat /dev/fd/63 - ++ echo 'crunchy_debug: "true"' ++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b ++ cut -d: -f2 ++ cut -d: -f1 ++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-721-a67b3475b"#g' + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml metadata.namespace self-healing-10347 + yq w -d3 - 'subjects[0].namespace' self-healing-10347 + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install + yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-deployer ++ cat /tmp/tmp.zr9HWJrZwl/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "self-healing-10347" namespace_mode: "disabled" pgo_image_prefix: "perconalab/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "PR-721-a67b3475b" pgo_operator_namespace: "self-healing-10347"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.zr9HWJrZwl/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.QMKPwWvpbJ ++ mktemp + local LAST_ERR=/tmp/tmp.TK4lIIVILz + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl apply -f /tmp/tmp.zr9HWJrZwl/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.QMKPwWvpbJ serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.TK4lIIVILz + rm /tmp/tmp.QMKPwWvpbJ /tmp/tmp.TK4lIIVILz + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=self-healing-10347 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.zr9HWJrZwl/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.z9AB47KtN6 ++ mktemp + local LAST_ERR=/tmp/tmp.jKXFCt7FYv + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete -f /tmp/tmp.zr9HWJrZwl/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.z9AB47KtN6 serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.jKXFCt7FYv + rm /tmp/tmp.z9AB47KtN6 /tmp/tmp.jKXFCt7FYv + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tUBjDPFQlr +++ mktemp ++ local LAST_ERR=/tmp/tmp.qjCdOiquQd ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.tUBjDPFQlr ++ cat /tmp/tmp.qjCdOiquQd ++ rm /tmp/tmp.tUBjDPFQlr /tmp/tmp.qjCdOiquQd ++ return 0 + wait_pod postgres-operator-77b45b447d-n5q66 + local pod=postgres-operator-77b45b447d-n5q66 + local ns= + set +o xtrace postgres-operator-77b45b447d-n5q66true + [[ -n '' ]] + deploy_chaos_mesh self-healing-10347 + local chaos_mesh_ns=self-healing-10347 + desc 'install chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- install chaos-mesh ----------------------------------------------------------------------------------- ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' + local old_cm_namespace=NAMESPACE + '[' NAMESPACE '!=' NAMESPACE ']' + helm repo remove chaos-mesh Error: no repo named "chaos-mesh" found + : + helm repo add chaos-mesh https://charts.chaos-mesh.org "chaos-mesh" has been added to your repositories + case ${PLATFORM} in + version_gt 1.19 ++ echo '1.26 >= 1.19' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=self-healing-10347 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version v2.0.4 --set clusterScoped=false --set controllerManager.targetNamespace=self-healing-10347 NAME: chaos-mesh LAST DEPLOYED: Tue Apr 23 16:35:18 2024 NAMESPACE: self-healing-10347 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: 1. Make sure chaos-mesh components are running kubectl get pods --namespace self-healing-10347 -l app.kubernetes.io/instance=chaos-mesh + sleep 10 + desc 'start cluster' + set +o xtrace ----------------------------------------------------------------------------------- start cluster ----------------------------------------------------------------------------------- + cluster=some-name + spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local backup=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + local namespace=self-healing-10347 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name self-healing-10347 + local cluster=some-name + local namespace=self-healing-10347 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml + yq w - stringData.some-name some-name_pass + yq w - metadata.name some-name-users-secret + kubectl_bin -n self-healing-10347 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.D9ylIfoguI ++ mktemp + local LAST_ERR=/tmp/tmp.mc00ZZrQpJ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl -n self-healing-10347 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.D9ylIfoguI secret/some-name-users-secret created + cat /tmp/tmp.mc00ZZrQpJ + rm /tmp/tmp.D9ylIfoguI /tmp/tmp.mc00ZZrQpJ + return 0 + yq w - metadata.name some-name-pmm-secret + kubectl_bin -n self-healing-10347 apply -f - + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.HGd3aJPHq5 ++ mktemp + local LAST_ERR=/tmp/tmp.vBMwtkKPbL + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl -n self-healing-10347 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.HGd3aJPHq5 secret/some-name-pmm-secret created + cat /tmp/tmp.vBMwtkKPbL + rm /tmp/tmp.HGd3aJPHq5 /tmp/tmp.vBMwtkKPbL + return 0 + kubectl_bin -n self-healing-10347 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.AKnzEtWG9k ++ mktemp + local LAST_ERR=/tmp/tmp.QTzNfngJds + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl -n self-healing-10347 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.AKnzEtWG9k deployment.apps/pg-client created + cat /tmp/tmp.QTzNfngJds + rm /tmp/tmp.AKnzEtWG9k /tmp/tmp.QTzNfngJds + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name false false false false false false false false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml + local name=some-name + local backup=false + local restore_from=false + local restore_opts=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name + yq w - metadata.labels.crunchy-pgha-scope some-name + yq w - metadata.labels.name some-name + yq w - metadata.labels.deployment-name some-name + yq w - spec.clustername some-name + yq w - spec.keepBackups false + yq w - spec.keepData false + yq w - spec.name some-name + yq w - spec.user some-name + yq w - spec.database some-name + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.pmm.image perconalab/pmm-client:2.41.0 + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - metadata.labels.pg-cluster some-name + yq w - metadata.name some-name + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.secretsName some-name-users-secret + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + case ${podAntiAffinity} in + case ${backup} in + [[ false != \f\a\l\s\e ]] + [[ false != \f\a\l\s\e ]] + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.zr9HWJrZwl/cr.yaml -n self-healing-10347 ++ mktemp + local LAST_OUT=/tmp/tmp.MJuK23XrCV ++ mktemp + local LAST_ERR=/tmp/tmp.oq6MiDQCmN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl apply -f /tmp/tmp.zr9HWJrZwl/cr.yaml -n self-healing-10347 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.MJuK23XrCV perconapgcluster.pg.percona.com/some-name created + cat /tmp/tmp.oq6MiDQCmN + rm /tmp/tmp.MJuK23XrCV /tmp/tmp.oq6MiDQCmN + return 0 + wait_deployment some-name-backrest-shared-repo + local name=some-name-backrest-shared-repo + local target_namespace=self-healing-10347 + sleep 10 + set +o xtrace some-name-backrest-shared-repo. + wait_deployment some-name + local name=some-name + local target_namespace=self-healing-10347 + sleep 10 + set +o xtrace some-name................. + [[ false == \g\c\s\+* ]] + wait_job_completion some-name-stanza-create false + local job=some-name-stanza-create + local mandatory=false + local ns=self-healing-10347 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl -n self-healing-10347 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' some-name-stanza-create+ retry=0 ++ kubectl_bin -n self-healing-10347 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name + local job=backrest-backup-some-name + local mandatory=true + local ns=self-healing-10347 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' backrest-backup-some-name+ retry=0 ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=self-healing-10347 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rwi6qjpO6W +++ mktemp ++ local LAST_ERR=/tmp/tmp.ASafZMmmrP ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.rwi6qjpO6W ++ cat /tmp/tmp.ASafZMmmrP ++ rm /tmp/tmp.rwi6qjpO6W /tmp/tmp.ASafZMmmrP ++ return 0 + status_query='kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.94z4kpXKGq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vxIz5EMDyy +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.94z4kpXKGq +++ cat /tmp/tmp.vxIz5EMDyy +++ rm /tmp/tmp.94z4kpXKGq /tmp/tmp.vxIz5EMDyy +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=self-healing-10347 +++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XA4bj33ZEn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EnfJZk9tq3 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.XA4bj33ZEn +++ cat /tmp/tmp.EnfJZk9tq3 +++ rm /tmp/tmp.XA4bj33ZEn /tmp/tmp.EnfJZk9tq3 +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.q3bAGognSX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Sdi74FzAwL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.q3bAGognSX +++ cat /tmp/tmp.Sdi74FzAwL +++ rm /tmp/tmp.q3bAGognSX /tmp/tmp.Sdi74FzAwL +++ return 0 +++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.S6wpZ0yfCC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.txU16qbktI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.S6wpZ0yfCC +++ cat /tmp/tmp.txU16qbktI +++ rm /tmp/tmp.S6wpZ0yfCC /tmp/tmp.txU16qbktI +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.HjVd14PnTQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2ea9PVAxUq ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.HjVd14PnTQ ++++ cat /tmp/tmp.2ea9PVAxUq ++++ rm /tmp/tmp.HjVd14PnTQ /tmp/tmp.2ea9PVAxUq ++++ return 0 +++ kubectl_bin -n self-healing-10347 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PkzHeDyr02 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cU1hP2Hc2K +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.PkzHeDyr02 +++ cat /tmp/tmp.cU1hP2Hc2K +++ rm /tmp/tmp.PkzHeDyr02 /tmp/tmp.cU1hP2Hc2K +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.self-healing-10347 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QRB69mZrVX +++ mktemp ++ local LAST_ERR=/tmp/tmp.1ohgFXeQXH ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.QRB69mZrVX ++ cat /tmp/tmp.1ohgFXeQXH ++ rm /tmp/tmp.QRB69mZrVX /tmp/tmp.1ohgFXeQXH ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.nVMEjAuAXB ++ mktemp + local LAST_ERR=/tmp/tmp.20yDd5yGrE + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.nVMEjAuAXB + cat /tmp/tmp.20yDd5yGrE + rm /tmp/tmp.nVMEjAuAXB /tmp/tmp.20yDd5yGrE + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=self-healing-10347 +++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lW5fyeTUV9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ypqSFsP29v +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.lW5fyeTUV9 +++ cat /tmp/tmp.ypqSFsP29v +++ rm /tmp/tmp.lW5fyeTUV9 /tmp/tmp.ypqSFsP29v +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rvgWJPajZm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Fre4wPD9mp +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.rvgWJPajZm +++ cat /tmp/tmp.Fre4wPD9mp +++ rm /tmp/tmp.rvgWJPajZm /tmp/tmp.Fre4wPD9mp +++ return 0 +++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uuGJUibdg8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9qxuwTjECa +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uuGJUibdg8 +++ cat /tmp/tmp.9qxuwTjECa +++ rm /tmp/tmp.uuGJUibdg8 /tmp/tmp.9qxuwTjECa +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.D3ex5lnvb3 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.lYt4YJdXs3 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.D3ex5lnvb3 ++++ cat /tmp/tmp.lYt4YJdXs3 ++++ rm /tmp/tmp.D3ex5lnvb3 /tmp/tmp.lYt4YJdXs3 ++++ return 0 +++ kubectl_bin -n self-healing-10347 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.a4rmBRIWTi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dzjXxQmrbr +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.a4rmBRIWTi +++ cat /tmp/tmp.dzjXxQmrbr +++ rm /tmp/tmp.a4rmBRIWTi /tmp/tmp.dzjXxQmrbr +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.self-healing-10347 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cNeKBu6cnD +++ mktemp ++ local LAST_ERR=/tmp/tmp.A0LQXiJ56S ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.cNeKBu6cnD ++ cat /tmp/tmp.A0LQXiJ56S ++ rm /tmp/tmp.cNeKBu6cnD /tmp/tmp.A0LQXiJ56S ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.hpirAnrchz ++ mktemp + local LAST_ERR=/tmp/tmp.z2B82sqFeh + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.hpirAnrchz + cat /tmp/tmp.z2B82sqFeh + rm /tmp/tmp.hpirAnrchz /tmp/tmp.z2B82sqFeh + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=self-healing-10347 +++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VcUicRtoI2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZJBPpNuuEe +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.VcUicRtoI2 +++ cat /tmp/tmp.ZJBPpNuuEe +++ rm /tmp/tmp.VcUicRtoI2 /tmp/tmp.ZJBPpNuuEe +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.U7Wyifemqg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8NYMiDOFeA +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.U7Wyifemqg +++ cat /tmp/tmp.8NYMiDOFeA +++ rm /tmp/tmp.U7Wyifemqg /tmp/tmp.8NYMiDOFeA +++ return 0 +++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7YozYPVpG7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7Urav5mSUw +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7YozYPVpG7 +++ cat /tmp/tmp.7Urav5mSUw +++ rm /tmp/tmp.7YozYPVpG7 /tmp/tmp.7Urav5mSUw +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.W6VCm12C30 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.eOX6wjEbV4 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++++ kubectl -n self-healing-10347 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.W6VCm12C30 ++++ cat /tmp/tmp.eOX6wjEbV4 ++++ rm /tmp/tmp.W6VCm12C30 /tmp/tmp.eOX6wjEbV4 ++++ return 0 +++ kubectl_bin -n self-healing-10347 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5KraIooUQH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pdDI8MnUN2 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl -n self-healing-10347 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.5KraIooUQH +++ cat /tmp/tmp.pdDI8MnUN2 +++ rm /tmp/tmp.5KraIooUQH /tmp/tmp.pdDI8MnUN2 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.self-healing-10347 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GxLQjYi7e3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.HsYuv6DnAw ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.GxLQjYi7e3 ++ cat /tmp/tmp.HsYuv6DnAw ++ rm /tmp/tmp.GxLQjYi7e3 /tmp/tmp.HsYuv6DnAw ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.BJryTbwYDN ++ mktemp + local LAST_ERR=/tmp/tmp.3bvrtaYgBq + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.BJryTbwYDN + cat /tmp/tmp.3bvrtaYgBq + rm /tmp/tmp.BJryTbwYDN /tmp/tmp.3bvrtaYgBq + return 0 + sleep 10 ++ get_pod_by_role some-name master name ++ local cluster=some-name ++ local role=master ++ local parameter=name ++ case ${parameter} in ++ local 'jsonpath={.items[].metadata.name}' +++ kubectl_bin get pods --selector=role=master,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JMiLWSD6Lr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SEltC4qPUt +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get pods --selector=role=master,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.JMiLWSD6Lr +++ cat /tmp/tmp.SEltC4qPUt +++ rm /tmp/tmp.JMiLWSD6Lr /tmp/tmp.SEltC4qPUt +++ return 0 ++ echo some-name-696f799f57-mwgfl + kill_pod some-name-696f799f57-mwgfl + local pod=some-name-696f799f57-mwgfl + cat /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/chaos-pod-kill.yml + yq w - metadata.name chaos-cluster-pod-kill-15166 + yq w - metadata.namespace self-healing-10347 + yq w - 'spec.selector.pods.self-healing-10347[0]' some-name-696f799f57-mwgfl + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.rvAkVsft2f ++ mktemp + local LAST_ERR=/tmp/tmp.RLJeZb4D1k + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.rvAkVsft2f podchaos.chaos-mesh.org/chaos-cluster-pod-kill-15166 created + cat /tmp/tmp.RLJeZb4D1k + rm /tmp/tmp.rvAkVsft2f /tmp/tmp.RLJeZb4D1k + return 0 + sleep 5 + wait_deployment some-name + local name=some-name + local target_namespace=self-healing-10347 + sleep 10 + set +o xtrace some-name... + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jje5v65BiB +++ mktemp ++ local LAST_ERR=/tmp/tmp.m7pN7OAyEO ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.jje5v65BiB ++ cat /tmp/tmp.m7pN7OAyEO ++ rm /tmp/tmp.jje5v65BiB /tmp/tmp.m7pN7OAyEO ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.hPKu886d6T ++ mktemp + local LAST_ERR=/tmp/tmp.22L0q2yw28 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.hPKu886d6T + cat /tmp/tmp.22L0q2yw28 + rm /tmp/tmp.hPKu886d6T /tmp/tmp.22L0q2yw28 + return 0 + '[' '!' -s /tmp/tmp.zr9HWJrZwl/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-1.sql /tmp/tmp.zr9HWJrZwl/select-1.sql ++ get_pod_by_role some-name master name ++ local cluster=some-name ++ local role=master ++ local parameter=name ++ case ${parameter} in ++ local 'jsonpath={.items[].metadata.name}' +++ kubectl_bin get pods --selector=role=master,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ibi82BQ0KN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IOcabLakFg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get pods --selector=role=master,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Ibi82BQ0KN +++ cat /tmp/tmp.IOcabLakFg +++ rm /tmp/tmp.Ibi82BQ0KN /tmp/tmp.IOcabLakFg +++ return 0 ++ echo some-name-repl1-7987df9b7-hckvh + failure_pod some-name-repl1-7987df9b7-hckvh + local pod=some-name-repl1-7987df9b7-hckvh + yq w - metadata.name chaos-cluster-pod-failure-32190 + yq w - metadata.namespace self-healing-10347 + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/chaos-pod-failure.yml ++ mktemp + yq w - 'spec.selector.pods.self-healing-10347[0]' some-name-repl1-7987df9b7-hckvh + local LAST_OUT=/tmp/tmp.EYI3iwGGdp ++ mktemp + local LAST_ERR=/tmp/tmp.WZ5KgzrDoL + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.EYI3iwGGdp podchaos.chaos-mesh.org/chaos-cluster-pod-failure-32190 created + cat /tmp/tmp.WZ5KgzrDoL + rm /tmp/tmp.EYI3iwGGdp /tmp/tmp.WZ5KgzrDoL + return 0 + sleep 10 + sleep 30 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I6Valjf1kz +++ mktemp ++ local LAST_ERR=/tmp/tmp.UniJYkXTmq ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.I6Valjf1kz ++ cat /tmp/tmp.UniJYkXTmq ++ rm /tmp/tmp.I6Valjf1kz /tmp/tmp.UniJYkXTmq ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.kdrgG2T3t9 ++ mktemp + local LAST_ERR=/tmp/tmp.IDXFevbA5V + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.kdrgG2T3t9 + cat /tmp/tmp.IDXFevbA5V + rm /tmp/tmp.kdrgG2T3t9 /tmp/tmp.IDXFevbA5V + return 0 + '[' '!' -s /tmp/tmp.zr9HWJrZwl/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-1.sql /tmp/tmp.zr9HWJrZwl/select-1.sql + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' ++ mktemp + local LAST_OUT=/tmp/tmp.NzZBtBmbOc ++ mktemp + local LAST_ERR=/tmp/tmp.UQdQ0QaHth + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.NzZBtBmbOc perconapgcluster.pg.percona.com/some-name patched (no change) + cat /tmp/tmp.UQdQ0QaHth + rm /tmp/tmp.NzZBtBmbOc /tmp/tmp.UQdQ0QaHth + return 0 + sleep 5 + wait_deployment some-name-repl1 + local name=some-name-repl1 + local target_namespace=self-healing-10347 + sleep 10 + set +o xtrace some-name-repl1 + wait_deployment some-name-repl2 + local name=some-name-repl2 + local target_namespace=self-healing-10347 + sleep 10 + set +o xtrace some-name-repl2 ++ get_pod_by_role some-name replica name repl1 ++ local cluster=some-name ++ local role=replica ++ local parameter=name ++ case ${parameter} in ++ local 'jsonpath={.items[].metadata.name}' +++ kubectl_bin get pods --selector=role=replica,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fhWpnuBkhb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nMIRGsSKpG +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get pods --selector=role=replica,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.fhWpnuBkhb +++ cat /tmp/tmp.nMIRGsSKpG +++ rm /tmp/tmp.fhWpnuBkhb /tmp/tmp.nMIRGsSKpG +++ return 0 ++ echo some-name-696f799f57-cm4r8 + kill_pod some-name-696f799f57-cm4r8 + local pod=some-name-696f799f57-cm4r8 + yq w - metadata.name chaos-cluster-pod-kill-26608 + yq w - metadata.namespace self-healing-10347 + kubectl_bin apply -f - + yq w - 'spec.selector.pods.self-healing-10347[0]' some-name-696f799f57-cm4r8 + cat /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/chaos-pod-kill.yml ++ mktemp + local LAST_OUT=/tmp/tmp.2hDAfXCbP4 ++ mktemp + local LAST_ERR=/tmp/tmp.rKPb14vCoU + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2hDAfXCbP4 podchaos.chaos-mesh.org/chaos-cluster-pod-kill-26608 created + cat /tmp/tmp.rKPb14vCoU + rm /tmp/tmp.2hDAfXCbP4 /tmp/tmp.rKPb14vCoU + return 0 + sleep 5 + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DOiGYnkSgm +++ mktemp ++ local LAST_ERR=/tmp/tmp.3TE3aAYmZQ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.DOiGYnkSgm ++ cat /tmp/tmp.3TE3aAYmZQ ++ rm /tmp/tmp.DOiGYnkSgm /tmp/tmp.3TE3aAYmZQ ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.s6IF7xgZe7 ++ mktemp + local LAST_ERR=/tmp/tmp.UPCbx1nYOS + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.s6IF7xgZe7 + cat /tmp/tmp.UPCbx1nYOS + rm /tmp/tmp.s6IF7xgZe7 /tmp/tmp.UPCbx1nYOS + return 0 + compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local command_id=select-2 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-2.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a0PDpUwFWu +++ mktemp ++ local LAST_ERR=/tmp/tmp.bzxiD3ki9F ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.a0PDpUwFWu ++ cat /tmp/tmp.bzxiD3ki9F ++ rm /tmp/tmp.a0PDpUwFWu /tmp/tmp.bzxiD3ki9F ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.epOpuJpGyQ ++ mktemp + local LAST_ERR=/tmp/tmp.nHpLX6LIMN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.epOpuJpGyQ + cat /tmp/tmp.nHpLX6LIMN + rm /tmp/tmp.epOpuJpGyQ /tmp/tmp.nHpLX6LIMN + return 0 + '[' '!' -s /tmp/tmp.zr9HWJrZwl/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-2.sql /tmp/tmp.zr9HWJrZwl/select-2.sql + wait_deployment some-name-repl1 + local name=some-name-repl1 + local target_namespace=self-healing-10347 + sleep 10 + set +o xtrace some-name-repl1 ++ get_pod_by_role some-name replica IP repl1 ++ local cluster=some-name ++ local role=replica ++ local parameter=IP ++ case ${parameter} in ++ local 'jsonpath={.items[].status.podIP}' +++ kubectl_bin get pods --selector=role=replica,pg-cluster=some-name -o 'jsonpath={.items[].status.podIP}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yhqm0JsR5t ++++ mktemp +++ local LAST_ERR=/tmp/tmp.M8v0ILZDmO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get pods --selector=role=replica,pg-cluster=some-name -o 'jsonpath={.items[].status.podIP}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.yhqm0JsR5t +++ cat /tmp/tmp.M8v0ILZDmO +++ rm /tmp/tmp.yhqm0JsR5t /tmp/tmp.M8v0ILZDmO +++ return 0 ++ echo 10.55.248.11 + repl1_pod_ip=10.55.248.11 + compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10-55-248-11.self-healing-10347 '' .pod.cluster.local + local command_id=select-2 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10-55-248-11.self-healing-10347 + local postfix= + local uri_suffix=.pod.cluster.local + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-2.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10-55-248-11.self-healing-10347 postgres .pod.cluster.local + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10-55-248-11.self-healing-10347 + local driver=postgres + local suffix=.pod.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2BJAs8SSLr +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fq8RXF7lOQ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.2BJAs8SSLr ++ cat /tmp/tmp.Fq8RXF7lOQ ++ rm /tmp/tmp.2BJAs8SSLr /tmp/tmp.Fq8RXF7lOQ ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10-55-248-11.self-healing-10347.pod.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.lR1V0yx0LB ++ mktemp + local LAST_ERR=/tmp/tmp.fmoY2ZIOsA + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10-55-248-11.self-healing-10347.pod.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.lR1V0yx0LB + cat /tmp/tmp.fmoY2ZIOsA + rm /tmp/tmp.lR1V0yx0LB /tmp/tmp.fmoY2ZIOsA + return 0 + '[' '!' -s /tmp/tmp.zr9HWJrZwl/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-2.sql /tmp/tmp.zr9HWJrZwl/select-2.sql ++ get_pod_by_role some-name replica name repl1 ++ local cluster=some-name ++ local role=replica ++ local parameter=name ++ case ${parameter} in ++ local 'jsonpath={.items[].metadata.name}' +++ kubectl_bin get pods --selector=role=replica,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Wbup9UdN5z ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ySAPIGUdEH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get pods --selector=role=replica,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Wbup9UdN5z +++ cat /tmp/tmp.ySAPIGUdEH +++ rm /tmp/tmp.Wbup9UdN5z /tmp/tmp.ySAPIGUdEH +++ return 0 ++ echo some-name-696f799f57-qr9vz + network_loss some-name-696f799f57-qr9vz + local pod=some-name-696f799f57-qr9vz + cat /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/chaos-network-loss.yml + yq w - metadata.namespace self-healing-10347 + yq w - 'spec.selector.pods.self-healing-10347[0]' some-name-696f799f57-qr9vz + kubectl_bin apply -f - ++ mktemp + yq w - metadata.name chaos-cluster-network-loss-32476 + local LAST_OUT=/tmp/tmp.8oIZZqWQtq ++ mktemp + local LAST_ERR=/tmp/tmp.tvbOFKnuia + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8oIZZqWQtq networkchaos.chaos-mesh.org/chaos-cluster-network-loss-32476 created + cat /tmp/tmp.tvbOFKnuia + rm /tmp/tmp.8oIZZqWQtq /tmp/tmp.tvbOFKnuia + return 0 + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k4EXRsNlnV +++ mktemp ++ local LAST_ERR=/tmp/tmp.YF0bqxMchL ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.k4EXRsNlnV ++ cat /tmp/tmp.YF0bqxMchL ++ rm /tmp/tmp.k4EXRsNlnV /tmp/tmp.YF0bqxMchL ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.mHxnHl17Sr ++ mktemp + local LAST_ERR=/tmp/tmp.Dd3dyTF4fN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.mHxnHl17Sr + cat /tmp/tmp.Dd3dyTF4fN + rm /tmp/tmp.mHxnHl17Sr /tmp/tmp.Dd3dyTF4fN + return 0 + compare_psql_cmd select-3 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local command_id=select-3 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-3.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.self-healing-10347 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A3YbB3MlD7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.iWTZ3LQXRQ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.A3YbB3MlD7 ++ cat /tmp/tmp.iWTZ3LQXRQ ++ rm /tmp/tmp.A3YbB3MlD7 /tmp/tmp.iWTZ3LQXRQ ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.pEyGVEplkw ++ mktemp + local LAST_ERR=/tmp/tmp.VFndyWsfoQ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.self-healing-10347.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.pEyGVEplkw + cat /tmp/tmp.VFndyWsfoQ + rm /tmp/tmp.pEyGVEplkw /tmp/tmp.VFndyWsfoQ + return 0 + '[' '!' -s /tmp/tmp.zr9HWJrZwl/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-3.sql /tmp/tmp.zr9HWJrZwl/select-3.sql + sleep 60 ++ get_pod_by_role some-name replica IP repl1 ++ local cluster=some-name ++ local role=replica ++ local parameter=IP ++ case ${parameter} in ++ local 'jsonpath={.items[].status.podIP}' +++ kubectl_bin get pods --selector=role=replica,pg-cluster=some-name -o 'jsonpath={.items[].status.podIP}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gSxZvHygeJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DWwHqlKxTn +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox +++ kubectl get pods --selector=role=replica,pg-cluster=some-name -o 'jsonpath={.items[].status.podIP}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.gSxZvHygeJ +++ cat /tmp/tmp.DWwHqlKxTn +++ rm /tmp/tmp.gSxZvHygeJ /tmp/tmp.DWwHqlKxTn +++ return 0 ++ echo 10.55.248.11 + repl1_pod_ip=10.55.248.11 + compare_psql_cmd select-3 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10-55-248-11.self-healing-10347 '' .pod.cluster.local + local command_id=select-3 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10-55-248-11.self-healing-10347 + local postfix= + local uri_suffix=.pod.cluster.local + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-3.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10-55-248-11.self-healing-10347 postgres .pod.cluster.local + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10-55-248-11.self-healing-10347 + local driver=postgres + local suffix=.pod.cluster.local ++ kubectl_bin -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FF0wIR0spe +++ mktemp ++ local LAST_ERR=/tmp/tmp.3osnXbExIg ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl -n self-healing-10347 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.FF0wIR0spe ++ cat /tmp/tmp.3osnXbExIg ++ rm /tmp/tmp.FF0wIR0spe /tmp/tmp.3osnXbExIg ++ return 0 + local client_container=pg-client-76df9df876-qc6pf + kubectl_bin exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10-55-248-11.self-healing-10347.pod.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.ni72zjMsfu ++ mktemp + local LAST_ERR=/tmp/tmp.6C6Um7uVlv + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl exec pg-client-76df9df876-qc6pf -n self-healing-10347 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10-55-248-11.self-healing-10347.pod.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ni72zjMsfu + cat /tmp/tmp.6C6Um7uVlv + rm /tmp/tmp.ni72zjMsfu /tmp/tmp.6C6Um7uVlv + return 0 + '[' '!' -s /tmp/tmp.zr9HWJrZwl/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/self-healing/compare/select-3.sql /tmp/tmp.zr9HWJrZwl/select-3.sql + kubectl delete NetworkChaos --all networkchaos.chaos-mesh.org "chaos-cluster-network-loss-32476" deleted + kubectl delete PodChaos --all podchaos.chaos-mesh.org "chaos-cluster-pod-failure-32190" deleted podchaos.chaos-mesh.org "chaos-cluster-pod-kill-15166" deleted podchaos.chaos-mesh.org "chaos-cluster-pod-kill-26608" deleted + sleep 5 + destroy_chaos_mesh self-healing-10347 + local chaos_mesh_ns=self-healing-10347 + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + helm del chaos-mesh --namespace self-healing-10347 release "chaos-mesh" uninstalled + destroy self-healing-10347 + grep -v level=info ++ get_operator_pod + grep -v level=debug ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp + grep -v 'Getting pods from source' + sort -u + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' ++ local LAST_OUT=/tmp/tmp.ifblQG8zxI + tee /tmp/tmp.zr9HWJrZwl/operator.log + grep -v 'Getting tasks for pod' +++ mktemp ++ local LAST_ERR=/tmp/tmp.ACOPzxMGd9 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ifblQG8zxI ++ cat /tmp/tmp.ACOPzxMGd9 ++ rm /tmp/tmp.ifblQG8zxI /tmp/tmp.ACOPzxMGd9 ++ return 0 + kubectl_bin logs postgres-operator-77b45b447d-n5q66 -c operator ++ mktemp + local LAST_OUT=/tmp/tmp.85ylJahSB7 ++ mktemp + local LAST_ERR=/tmp/tmp.5XdaX4vuM1 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl logs postgres-operator-77b45b447d-n5q66 -c operator + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.85ylJahSB7 + cat /tmp/tmp.5XdaX4vuM1 + rm /tmp/tmp.85ylJahSB7 /tmp/tmp.5XdaX4vuM1 + return 0 ] { } }, ] } }, { } }, ] { } }, { } ] } { } }, ] } ] { } }, }, { ] { } }, ] ] ], { } }, }, { }, { },{ }] }], ] { } }, , ], { } }, }], { } }, }], } ], } }, } }, } }, { } }, }, 2024/04/23 16:35:41 INF 1 (localhost:4150) connecting to nsqd 2024/04/23 16:36:45 INF 2 (localhost:4150) connecting to nsqd 2024/04/23 16:36:48 INF 3 (localhost:4150) connecting to nsqd 2024/04/23 16:36:52 INF 4 (localhost:4150) connecting to nsqd 2024/04/23 16:37:21 INF 5 (localhost:4150) connecting to nsqd 2024/04/23 16:37:21 INF 6 (localhost:4150) connecting to nsqd 2024/04/23 16:37:21 INF 7 (localhost:4150) connecting to nsqd "accessModes": [ "affinity": "affinity": { "affinity": { "podAntiAffinity": { "allowPrivilegeEscalation": false, "annotations": {"keep-backups":"false","keep-data":"false"}, "apiVersion": "apps/v1", "apiVersion": "batch/v1", "apiVersion": "v1", "apiVersion": "v1", "apiVersion": "v1", "backrest-command": "backup", "backrest-command": "stanza-create", "claimName": "some-name-pgbr-repo" "command": [ "configMap": { "configMap": { "configMapKeyRef": { "containerName": "database", "containerPort": 2022, "containerPort": 5432, "containerPort": 8009, "containers": [ "containers": [{ "cpu": "1", "cpu": "200m", "cpu": "500m", "crunchydata" "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "defaultMode": 288 "defaultMode": 288, "defaultMode": 288, "defaultMode": 420, "deployment-name": "some-name" "deployment-name": "some-name-repl1" "deployment-name": "some-name-repl2" "divisor": "1m", "dnsPolicy": "ClusterFirst" "downwardAPI": { "emptyDir": { "emptyDir": { "emptyDir": { "emptyDir": { "env": [ "env": [{ "exec": { "fieldPath": "metadata.annotations" "fieldPath": "metadata.labels" "fieldPath": "metadata.labels['crunchy-pgha-scope']" "fieldPath": "metadata.namespace" "fieldRef": { "fieldRef": { "fsGroup": 2, I0423 16:34:38.638006 1 shared_informer.go:240] Waiting for caches to sync for namespace I0423 16:34:38.738320 1 shared_informer.go:247] Caches are synced for namespace I0423 16:34:38.856134 1 shared_informer.go:240] Waiting for caches to sync for self-healing-10347 I0423 16:34:38.956719 1 shared_informer.go:247] Caches are synced for self-healing-10347 "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer", "image": "perconalab/percona-postgresql-operator:main-ppg14-postgres-ha", "imagePullPolicy": "IfNotPresent" "imagePullPolicy": "IfNotPresent", "initialDelaySeconds": 15 "initialDelaySeconds": 30, "items": [ "items": [ "items": [ "key": "config", "key": "config", "key": "crunchy-pgbouncer", "key": "init" "key": "password" "key": "pg-cluster", "key": "pgo-backrest", "key": "pgo-backrest-job", "key": "pgo-pg-database", "key": "pg-pod-anti-affinity", "key": "tls.crt", "key": "tls.key", "key": "vendor", "kind": "Deployment", "kind": "Job", "kind": "PersistentVolumeClaim", "kind": "Service", "labels": { "labels": { "labels": { "labelSelector": { "livenessProbe": { "matchExpressions": [ "matchLabels": { "matchLabels": { "maxSurge": 1 "maxUnavailable": 1, "medium": "Memory", "medium": "Memory" "medium": "Memory", "medium": "Memory", "medium": "Memory", "memory": "128Mi" "memory": "256Mi" "memory": "48Mi" "metadata": { "metadata": { "metadata": { "mountPath": "/backrestrepo", "mountPath": "/dev/shm", "mountPath": "/etc/pgbackrest" "mountPath": "/etc/pgbackrest/conf.d", "mountPath": "/etc/podinfo", "mountPath": "/etc/ssh", "mountPath": "/etc/ssh", "mountPath": "/home/pgbackrest" "mountPath": "/pgconf", "mountPath": "/pgconf/", "mountPath": "/pgconf/pgreplicator", "mountPath": "/pgconf/pgsuper", "mountPath": "/pgconf/pguser", "mountPath": "/pgconf/tls", "mountPath": "/pgconf/tls/pgbouncer", "mountPath": "/pgconf/tls-replication", "mountPath": "/pgdata", "mountPath": "/sshd", "mountPath": "/sshd", "mountPath": "/tmp", "mountPath": "/tmp", "mountPath": "/tmp" "name": "backrest", "name": "backrest-backup-some-name", "name": "backrestrepo", "name": "backrestrepo", "name": "BACKREST_SKIP_CREATE_STANZA", "name": "cluster1-replication-ssl-keypair", "name": "cluster1-ssl-ca" "name": "cluster1-ssl-ca" "name": "cluster1-ssl-keypair" "name": "cluster1-ssl-keypair" "name": "COMMAND", "name": "COMMAND_OPTS", "name": "database", "name": "dshm" "name": "dshm", "name": "ENABLE_SSHD", "name": "LD_PRELOAD", "name": "LD_PRELOAD", "name": "MODE", "name": "NAMESPACE", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_PASSWD", "name": "NSS_WRAPPER_PASSWD", "name": "patroni" "name": "PATRONI_KUBERNETES_LABELS", "name": "PATRONI_KUBERNETES_NAMESPACE", "name": "PATRONI_KUBERNETES_SCOPE_LABEL", "name": "PATRONI_LOG_LEVEL", "name": "PATRONI_POSTGRESQL_DATA_DIR", "name": "PATRONI_SCOPE", "name": "pgbackrest" "name": "pgbackrest-conf", "name": "pgbackrest-conf", "name": "pgbackrest-config" "name": "pgbackrest-config", "name": "PGBACKREST_DB_HOST", "name": "PGBACKREST_DB_PATH", "name": "PGBACKREST_DB_PATH", "name": "pgbackrest-home", "name": "pgbackrest-home", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_REPO1_HOST", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_STANZA", "name": "PGBACKREST_STANZA", "name": "pgbouncer" "name": "pgbouncer", "name": "pgbouncer-conf", "name": "pgbouncer-conf", "name": "pgconf-volume" "name": "pgconf-volume", "name": "pgdata", "name": "pgdata", "name": "PGHA_DATABASE", "name": "PGHA_INIT", "name": "PGHA_PASSWORD_TYPE", "name": "PGHA_PGBACKREST", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_S3_VERIFY_TLS", "name": "PGHA_PG_PORT", "name": "PGHA_REPLICA_REINIT_ON_START_FAIL", "name": "PGHA_STANDBY", "name": "PGHA_SYNC_REPLICATION", "name": "PGHA_TLS_ENABLED", "name": "PGHA_TLS_ONLY", "name": "PGHA_USER", "name": "PGHOST", "name": "PG_PASSWORD", "name": "PG_PRIMARY_SERVICE_NAME", "name": "PITR_TARGET", "name": "podinfo" "name": "podinfo", "name": "PODNAME", "name": "postgres" "name": "primary-volume" "name": "primary-volume", "name": "report", "name": "root-volume" "name": "root-volume", "name": "some-name", "name": "some-name", "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo" "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer-cm" "name": "some-name-pgbouncer-secret", "name": "some-name-pgbouncer-secret", "name": "some-name-pgbr-repo", "name": "some-name","pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0" "name": "some-name-pgha-config", "name": "some-name-pgha-config", "name": "some-name-repl1", "name": "some-name-repl1", "name": "some-name-repl2", "name": "some-name-repl2", "name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2","pgo-version": "1.6.0" "name": "some-name-stanza-create", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "sshd", "name": "sshd", "name": "sshd", "name": "sshd", "name": "tls-pgbouncer" "name": "tls-pgbouncer", "name": "tls-replication" "name": "tls-replication", "name": "tls-server" "name": "tls-server", "name": "tmp" "name": "tmp" "name": "tmp", "name": "tmp", "name": "tmp", "name": "user-volume" "name": "user-volume", "nodePort": 0 "operator": "Exists" "operator": "In", "/opt/crunchy/bin/postgres-ha/health/pgha-liveness.sh" "/opt/crunchy/bin/postgres-ha/health/pgha-readiness.sh" "optional": true "path": "annotations" "path": "cpu_limit", "path": "cpu_request", "path": "labels" "path": "mem_limit", "path": "mem_request", "path": "ssh_config" "path": "ssh_config" "path": "tls-replication.crt" "path": "tls-replication.key" "periodSeconds": 15, "persistentVolumeClaim": { "persistentVolumeClaim":{"claimName":"some-name"} "persistentVolumeClaim":{"claimName":"some-name-repl1"} "persistentVolumeClaim":{"claimName":"some-name-repl2"} "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0","name": "some-name" "pg-cluster": "some-name","deployment-name": "some-name-repl1","pgo-version": "1.6.0","name": "some-name-replica" "pg-cluster": "some-name","deployment-name": "some-name-repl2","pgo-version": "1.6.0","name": "some-name-replica" "pgo-backrest-job": "true", "pgo-backrest-repo": "true" "pgo-backrest-repo": "true", "pgo-backrest-repo": "true" "pgo-backrest": "true", "pgo-pg-database": "true", "pgo-pg-database": "true", "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1" "pg-pod-anti-affinity": "preferred", "pg-pod-anti-affinity": "preferred", "podAffinityTerm": { "podAntiAffinity": { "podAntiAffinity": { "podAntiAffinity": { "port": 2022, "ports": [{ "ports": [{ "preferredDuringSchedulingIgnoredDuringExecution": [ "privileged": false, "projected": { "projected": { "projected": { "projected": { "sources": [] } "protocol": "TCP", "protocol": "TCP", "readinessProbe": { "readOnly": false "readOnly": false "readOnly": false "readOnlyRootFilesystem": true "readOnly": true "readOnly": true "readOnly": true "ReadWriteOnce" "replicas": 0, "replicas": 1, "replicas": 3, "requests": { "requests": { "resourceFieldRef": { "resource": "limits.cpu" "resource": "limits.memory" "resource": "requests.cpu" "resource": "requests.memory" "resources": { "resources": { "restartPolicy": "Always", "restartPolicy": "Never" "rollingUpdate": { "runAsNonRoot": true "secret": { "secret": { "secret": { "secret": { "secret": { "secretKeyRef": { "secretName": "some-name-backrest-repo-config" "secretName": "some-name-backrest-repo-config", "secretName": "some-name-backrest-repo-config" "secretName": "some-name-backrest-repo-config", "secretName": "some-name-postgres-secret" "secretName": "some-name-primaryuser-secret" "secretName": "some-name-some-name-secret" "securityContext": { "securityContext": { "securityContext": {}, "securityContext": {"fsGroup": 26,"supplementalGroups": [1001]}, "securityContext": {"runAsNonRoot":true,"fsGroup":26}, "selector": { "selector": { "serviceAccountName": "pgo-backrest", "serviceAccountName": "pgo-default", "serviceAccountName": "pgo-pg", "service-name": "some-name-backrest-shared-repo", "service-name": "some-name-backrest-shared-repo", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer", "sessionAffinity": "None" "sizeLimit": "128Ki" "sizeLimit": "16Mi" "sizeLimit": "1Mi" "sizeLimit": "2Mi" "sizeLimit": "64Mi" "sizeLimit": "64Mi" "some-name" "sources": [ "sources": [ "sources": [ "spec": { "spec": { "spec": { "storage": "1Gi" "strategy": { "targetPort": 2022, "template": { time="2024-04-23T16:34:38Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.6.0 time="2024-04-23T16:35:45Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T16:35:50Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-23T16:36:45Z" level=error msg="Error completing workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-23T16:36:45Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-23T16:37:22Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-23T16:37:22Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 "timeoutSeconds": 10 "topologyKey": "kubernetes.io/hostname" "type": "ClusterIP", "type": "RollingUpdate", "value": "" "value": "5432" "value": "5432" "value": "/backrestrepo/some-name-backrest-shared-repo" "value": "/backrestrepo/some-name-backrest-shared-repo" "value": "backup" "value": "crunchy-pgha-scope" "value": "db" "value": "db" "value": " --db-host=10.55.248.8 --db-path=/pgdata/some-name" "value": "false" "value": "false" "valueFrom": { "value": "INFO" "value": "pgbackrest" "value": "pgbackrest-repo" "value": "/pgdata/some-name" "value": "/pgdata/some-name" "value": "/pgdata/some-name-repl1" "value": "/pgdata/some-name-repl1" "value": "/pgdata/some-name-repl2" "value": "/pgdata/some-name-repl2" "value": "posix" "value": "posix" "value": "postgres" "values": [ "value": "some-name" "value": "some-name-backrest-shared-repo" "value": "some-name-backrest-shared-repo-6f8f947dbc-rtbmd" "value": "stanza-create" "value": "/tmp" "value": "/tmp" "value": "/tmp/nss_wrapper/pgbackrest-repo/group" "value": "/tmp/nss_wrapper/pgbackrest-repo/passwd" "value": "/tmp/nss_wrapper/postgres/group" "value": "/tmp/nss_wrapper/postgres/passwd" "value": "true" "value": "true" "value": "--type=full --db-host=10.55.248.8 --db-path=/pgdata/some-name" "value": "/usr/lib64/libnss_wrapper.so" "value": "/usr/lib64/libnss_wrapper.so" "value": "{vendor: \"crunchydata\"}" "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "volumeMounts": [ "volumeMounts": [{ "volumes": [ "volumes": [{ "weight": 1, + destroy_operator + kubectl_bin delete -f /tmp/tmp.zr9HWJrZwl/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.a35MKcffyY ++ mktemp + local LAST_ERR=/tmp/tmp.N9p1NjeU2S + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete -f /tmp/tmp.zr9HWJrZwl/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete -f /tmp/tmp.zr9HWJrZwl/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete -f /tmp/tmp.zr9HWJrZwl/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.a35MKcffyY + cat /tmp/tmp.N9p1NjeU2S Error from server (NotFound): error when deleting "/tmp/tmp.zr9HWJrZwl/operator.yaml": serviceaccounts "pgo-deployer-sa" not found Error from server (NotFound): error when deleting "/tmp/tmp.zr9HWJrZwl/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): error when deleting "/tmp/tmp.zr9HWJrZwl/operator.yaml": configmaps "pgo-deployer-cm" not found Error from server (NotFound): error when deleting "/tmp/tmp.zr9HWJrZwl/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found Error from server (NotFound): error when deleting "/tmp/tmp.zr9HWJrZwl/operator.yaml": jobs.batch "pgo-deploy" not found + rm /tmp/tmp.a35MKcffyY /tmp/tmp.N9p1NjeU2S + return 1 + true + yq w -d4 /tmp/tmp.zr9HWJrZwl/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall + kubectl_bin apply -f /tmp/tmp.zr9HWJrZwl/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.9KB5xULjyI ++ mktemp + local LAST_ERR=/tmp/tmp.HAEBlUxLmS + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl apply -f /tmp/tmp.zr9HWJrZwl/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.9KB5xULjyI serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.HAEBlUxLmS + rm /tmp/tmp.9KB5xULjyI /tmp/tmp.HAEBlUxLmS + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=self-healing-10347 + retry=0 + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n self-healing-10347 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.zr9HWJrZwl/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.2Cx7mkcQMe ++ mktemp + local LAST_ERR=/tmp/tmp.T3RXO9xQYl + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete -f /tmp/tmp.zr9HWJrZwl/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2Cx7mkcQMe serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.T3RXO9xQYl + rm /tmp/tmp.2Cx7mkcQMe /tmp/tmp.T3RXO9xQYl + return 0 + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + awk '{print$1}' ++ mktemp + xargs kubectl delete ns --grace-period=0 --force=true + local LAST_OUT=/tmp/tmp.SgrlLCvPmH ++ mktemp + local LAST_ERR=/tmp/tmp.2GkfVtBONO + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.SgrlLCvPmH + cat /tmp/tmp.2GkfVtBONO + rm /tmp/tmp.SgrlLCvPmH /tmp/tmp.2GkfVtBONO + return 0 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. namespace "self-healing-10347" force deleted + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.c1UJKmTC02 ++ mktemp + local LAST_ERR=/tmp/tmp.eQ56QspBwu + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.c1UJKmTC02 + cat /tmp/tmp.eQ56QspBwu Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.c1UJKmTC02 /tmp/tmp.eQ56QspBwu + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.pwnZHWP9AD ++ mktemp + local LAST_ERR=/tmp/tmp.6V11CRko9t + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.pwnZHWP9AD + cat /tmp/tmp.6V11CRko9t Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.pwnZHWP9AD /tmp/tmp.6V11CRko9t + return 1 + true + rm -rf /tmp/tmp.zr9HWJrZwl