++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/logs/recreate.log' Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/logs/recreate.log ++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: the server doesn't have a resource type "projects" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' ']' +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.26 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.14.4 ++ '[' v3 == v2 ']' + create_namespace recreate-4603 + local namespace=recreate-4603 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + cleanup_rbac + awk '{print$1}' + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.vGxKlRubd9 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.hRT2GHfQad + local LAST_ERR=/tmp/tmp.KYUoMfalaF + local exit_status=0 + local timeout=4 + set +o errexit ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.VW70FbyuLx + local exit_status=0 + local timeout=4 + set +o errexit + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get ns ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.vGxKlRubd9 + cat /tmp/tmp.KYUoMfalaF + rm /tmp/tmp.vGxKlRubd9 /tmp/tmp.KYUoMfalaF + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.hRT2GHfQad + cat /tmp/tmp.VW70FbyuLx Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.hRT2GHfQad /tmp/tmp.VW70FbyuLx + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + local LAST_OUT=/tmp/tmp.iNDEPUzhK1 ++ mktemp + local LAST_ERR=/tmp/tmp.i8Y27O4Afi + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.iNDEPUzhK1 + cat /tmp/tmp.i8Y27O4Afi Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.iNDEPUzhK1 /tmp/tmp.i8Y27O4Afi + return 1 + true + kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found + true + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin delete namespace recreate-4603 ++ mktemp + local LAST_OUT=/tmp/tmp.UejQyzXiVv ++ mktemp + local LAST_ERR=/tmp/tmp.rPbgOu7zWG + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete namespace recreate-4603 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete namespace recreate-4603 + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete namespace recreate-4603 + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.UejQyzXiVv + cat /tmp/tmp.rPbgOu7zWG Error from server (NotFound): namespaces "recreate-4603" not found + rm /tmp/tmp.UejQyzXiVv /tmp/tmp.rPbgOu7zWG + return 1 + : + wait_for_delete namespace/recreate-4603 + local res=namespace/recreate-4603 + set +o xtrace namespace/recreate-4603 - Error from server (NotFound): namespaces "recreate-4603" not found + kubectl_bin create namespace recreate-4603 ++ mktemp + local LAST_OUT=/tmp/tmp.qS8LboOH2o ++ mktemp + local LAST_ERR=/tmp/tmp.K1paQib7CW + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl create namespace recreate-4603 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.qS8LboOH2o namespace/recreate-4603 created + cat /tmp/tmp.K1paQib7CW + rm /tmp/tmp.qS8LboOH2o /tmp/tmp.K1paQib7CW + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.gHZAzz0b7K +++ mktemp ++ local LAST_ERR=/tmp/tmp.vJ5ybUaRG4 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.gHZAzz0b7K ++ cat /tmp/tmp.vJ5ybUaRG4 ++ rm /tmp/tmp.gHZAzz0b7K /tmp/tmp.vJ5ybUaRG4 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-723-7ed19e708-2-sandbox --namespace=recreate-4603 ++ mktemp + local LAST_OUT=/tmp/tmp.HKFi3iNJDU ++ mktemp + local LAST_ERR=/tmp/tmp.zxSmg5jq8J + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-723-7ed19e708-2-sandbox --namespace=recreate-4603 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.HKFi3iNJDU Context "gke_cloud-dev-112233_us-central1-c_jen-pg-723-7ed19e708-2-sandbox" modified. + cat /tmp/tmp.zxSmg5jq8J + rm /tmp/tmp.HKFi3iNJDU /tmp/tmp.zxSmg5jq8J + return 0 + deploy_operator + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml + local namespace_mode=disabled + local action=install + local namespace=recreate-4603 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml disabled install recreate-4603 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml + local namespace_mode=disabled + local operator_action=install + local namespace=recreate-4603 + local dont_send_telemetry=false + /usr/bin/sed -e 's#^namespace: .*#namespace: "recreate-4603"#g' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "recreate-4603"#g' + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml 'data[values.yaml]' + cat /dev/fd/63 - ++ echo 'crunchy_debug: "true"' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g' + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' ++ cut -d: -f2 ++ echo perconalab/percona-postgresql-operator:PR-723-7ed19e708 ++ cut -d: -f1 ++ echo perconalab/percona-postgresql-operator:PR-723-7ed19e708 + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-723-7ed19e708"#g' + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml metadata.namespace recreate-4603 + yq w -d3 - 'subjects[0].namespace' recreate-4603 + yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-723-7ed19e708-pgo-deployer + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install ++ cat /tmp/tmp.UXRFYGRGBI/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "recreate-4603" namespace_mode: "disabled" pgo_image_prefix: "perconalab/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "PR-723-7ed19e708" pgo_operator_namespace: "recreate-4603"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.UXRFYGRGBI/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.3rgGwSKUOP ++ mktemp + local LAST_ERR=/tmp/tmp.W98b5W4Jh4 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl apply -f /tmp/tmp.UXRFYGRGBI/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.3rgGwSKUOP serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.W98b5W4Jh4 + rm /tmp/tmp.3rgGwSKUOP /tmp/tmp.W98b5W4Jh4 + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=recreate-4603 + retry=0 + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.UXRFYGRGBI/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.gtGsaxtXQt ++ mktemp + local LAST_ERR=/tmp/tmp.rOtPqxeH4e + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete -f /tmp/tmp.UXRFYGRGBI/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.gtGsaxtXQt serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.rOtPqxeH4e + rm /tmp/tmp.gtGsaxtXQt /tmp/tmp.rOtPqxeH4e + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CAt1XTJoHZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.2Ym9ErluBe ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.CAt1XTJoHZ ++ cat /tmp/tmp.2Ym9ErluBe ++ rm /tmp/tmp.CAt1XTJoHZ /tmp/tmp.2Ym9ErluBe ++ return 0 + wait_pod postgres-operator-58d9cddfdb-82fnk + local pod=postgres-operator-58d9cddfdb-82fnk + local ns= + set +o xtrace postgres-operator-58d9cddfdb-82fnktrue + [[ -n '' ]] + cluster=some-name + spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/cr.yaml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/cr.yaml + local backup=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/client.yml + local namespace=recreate-4603 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name recreate-4603 + local cluster=some-name + local namespace=recreate-4603 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/users-secret.yaml + yq w - stringData.some-name some-name_pass + yq w - metadata.name some-name-users-secret + kubectl_bin -n recreate-4603 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Y2KKu1NQnp ++ mktemp + local LAST_ERR=/tmp/tmp.T3Sf1ScD6G + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Y2KKu1NQnp secret/some-name-users-secret created + cat /tmp/tmp.T3Sf1ScD6G + rm /tmp/tmp.Y2KKu1NQnp /tmp/tmp.T3Sf1ScD6G + return 0 + yq w - metadata.name some-name-pmm-secret + kubectl_bin -n recreate-4603 apply -f - + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/users-secret.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.qGj22oCmp6 ++ mktemp + local LAST_ERR=/tmp/tmp.8hL3EpLOBt + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.qGj22oCmp6 secret/some-name-pmm-secret created + cat /tmp/tmp.8hL3EpLOBt + rm /tmp/tmp.qGj22oCmp6 /tmp/tmp.8hL3EpLOBt + return 0 + kubectl_bin -n recreate-4603 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.cYpFy4wHo7 ++ mktemp + local LAST_ERR=/tmp/tmp.s8Zb1duW2I + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.cYpFy4wHo7 deployment.apps/pg-client created + cat /tmp/tmp.s8Zb1duW2I + rm /tmp/tmp.cYpFy4wHo7 /tmp/tmp.s8Zb1duW2I + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/cr.yaml some-name false false false false false false false false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/cr.yaml + local name=some-name + local backup=false + local restore_from=false + local restore_opts=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/cr.yaml metadata.annotations.current-primary some-name + yq w - metadata.labels.deployment-name some-name + yq w - metadata.labels.name some-name + yq w - metadata.labels.crunchy-pgha-scope some-name + yq w - spec.keepData false + yq w - spec.user some-name + yq w - spec.clustername some-name + yq w - spec.keepBackups false + yq w - spec.database some-name + yq w - metadata.labels.pg-cluster some-name + yq w - spec.name some-name + yq w - spec.secretsName some-name-users-secret + yq w - metadata.name some-name + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.pmm.image perconalab/pmm-client:2.41.0 + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + case ${podAntiAffinity} in + case ${backup} in + [[ false != \f\a\l\s\e ]] + [[ false != \f\a\l\s\e ]] + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.UXRFYGRGBI/cr.yaml -n recreate-4603 ++ mktemp + local LAST_OUT=/tmp/tmp.wmxi7n3fUG ++ mktemp + local LAST_ERR=/tmp/tmp.jtIpPollKo + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl apply -f /tmp/tmp.UXRFYGRGBI/cr.yaml -n recreate-4603 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.wmxi7n3fUG perconapgcluster.pg.percona.com/some-name created + cat /tmp/tmp.jtIpPollKo + rm /tmp/tmp.wmxi7n3fUG /tmp/tmp.jtIpPollKo + return 0 + wait_deployment some-name-backrest-shared-repo + local name=some-name-backrest-shared-repo + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-backrest-shared-repo... + wait_deployment some-name + local name=some-name + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name......... + [[ false == \g\c\s\+* ]] + wait_job_completion some-name-stanza-create false + local job=some-name-stanza-create + local mandatory=false + local ns=recreate-4603 + retry=0 + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' some-name-stanza-create+ retry=0 ++ kubectl_bin -n recreate-4603 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name + local job=backrest-backup-some-name + local mandatory=true + local ns=recreate-4603 + retry=0 + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' backrest-backup-some-name+ retry=0 ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=recreate-4603 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QLuPTHOFgf +++ mktemp ++ local LAST_ERR=/tmp/tmp.8UCL48qW6R ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.QLuPTHOFgf ++ cat /tmp/tmp.8UCL48qW6R ++ rm /tmp/tmp.QLuPTHOFgf /tmp/tmp.8UCL48qW6R ++ return 0 + status_query='kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2UJlOWfwWh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tcGa10vYrT +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2UJlOWfwWh +++ cat /tmp/tmp.tcGa10vYrT +++ rm /tmp/tmp.2UJlOWfwWh /tmp/tmp.tcGa10vYrT +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 1 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fGKq2Ix3Jy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zE2M6SsqCm +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.fGKq2Ix3Jy +++ cat /tmp/tmp.zE2M6SsqCm +++ rm /tmp/tmp.fGKq2Ix3Jy /tmp/tmp.zE2M6SsqCm +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jqclcGhdfX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.j3pPAJwYGg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.jqclcGhdfX +++ cat /tmp/tmp.j3pPAJwYGg +++ rm /tmp/tmp.jqclcGhdfX /tmp/tmp.j3pPAJwYGg +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NvksVVoOJ8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Vq72KcnE42 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.NvksVVoOJ8 +++ cat /tmp/tmp.Vq72KcnE42 +++ rm /tmp/tmp.NvksVVoOJ8 /tmp/tmp.Vq72KcnE42 +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.r2PoaubsND ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cm7Fx5kI9G +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.r2PoaubsND +++ cat /tmp/tmp.cm7Fx5kI9G +++ rm /tmp/tmp.r2PoaubsND /tmp/tmp.cm7Fx5kI9G +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.vTCPMne7FR +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.mAKR0jPCHq ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.vTCPMne7FR ++++ cat /tmp/tmp.mAKR0jPCHq ++++ rm /tmp/tmp.vTCPMne7FR /tmp/tmp.mAKR0jPCHq ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bxnLZDYVQY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tNYWbNvlpo +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bxnLZDYVQY +++ cat /tmp/tmp.tNYWbNvlpo +++ rm /tmp/tmp.bxnLZDYVQY /tmp/tmp.tNYWbNvlpo +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.recreate-4603 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eth7JLpzIz +++ mktemp ++ local LAST_ERR=/tmp/tmp.vqeeKbdWtH ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.eth7JLpzIz ++ cat /tmp/tmp.vqeeKbdWtH ++ rm /tmp/tmp.eth7JLpzIz /tmp/tmp.vqeeKbdWtH ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.KeGnWey1nm ++ mktemp + local LAST_ERR=/tmp/tmp.LaMRNcaZG6 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.KeGnWey1nm + cat /tmp/tmp.LaMRNcaZG6 + rm /tmp/tmp.KeGnWey1nm /tmp/tmp.LaMRNcaZG6 + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XedARKCZ8d ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cC7SSY4BRK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.XedARKCZ8d +++ cat /tmp/tmp.cC7SSY4BRK +++ rm /tmp/tmp.XedARKCZ8d /tmp/tmp.cC7SSY4BRK +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OBziGOsbYl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MREoWJCrYS +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.OBziGOsbYl +++ cat /tmp/tmp.MREoWJCrYS +++ rm /tmp/tmp.OBziGOsbYl /tmp/tmp.MREoWJCrYS +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ivDdLbnAYx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DodtrA51nK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ivDdLbnAYx +++ cat /tmp/tmp.DodtrA51nK +++ rm /tmp/tmp.ivDdLbnAYx /tmp/tmp.DodtrA51nK +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.NfTZzFQdrC +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.T3iRjIoAc3 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.NfTZzFQdrC ++++ cat /tmp/tmp.T3iRjIoAc3 ++++ rm /tmp/tmp.NfTZzFQdrC /tmp/tmp.T3iRjIoAc3 ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iMWdb7E5rK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pQEkWqb2wQ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.iMWdb7E5rK +++ cat /tmp/tmp.pQEkWqb2wQ +++ rm /tmp/tmp.iMWdb7E5rK /tmp/tmp.pQEkWqb2wQ +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.recreate-4603 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9wg2CFK4dI +++ mktemp ++ local LAST_ERR=/tmp/tmp.6GmPlbClxR ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.9wg2CFK4dI ++ cat /tmp/tmp.6GmPlbClxR ++ rm /tmp/tmp.9wg2CFK4dI /tmp/tmp.6GmPlbClxR ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.gBYcLM66Az ++ mktemp + local LAST_ERR=/tmp/tmp.TfiGfB6uiY + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.gBYcLM66Az + cat /tmp/tmp.TfiGfB6uiY + rm /tmp/tmp.gBYcLM66Az /tmp/tmp.TfiGfB6uiY + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.esShtLMk3q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rgzKJuPg0g +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.esShtLMk3q +++ cat /tmp/tmp.rgzKJuPg0g +++ rm /tmp/tmp.esShtLMk3q /tmp/tmp.rgzKJuPg0g +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.a6hhFs9wbQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.f8juW2z0mE +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.a6hhFs9wbQ +++ cat /tmp/tmp.f8juW2z0mE +++ rm /tmp/tmp.a6hhFs9wbQ /tmp/tmp.f8juW2z0mE +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dQpgyDuPtm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PWjxzerukh +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.dQpgyDuPtm +++ cat /tmp/tmp.PWjxzerukh +++ rm /tmp/tmp.dQpgyDuPtm /tmp/tmp.PWjxzerukh +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.p7P3WvBNzh +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ddHas9U6bU ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.p7P3WvBNzh ++++ cat /tmp/tmp.ddHas9U6bU ++++ rm /tmp/tmp.p7P3WvBNzh /tmp/tmp.ddHas9U6bU ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.566m0GimJp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CQxHbpZeQF +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.566m0GimJp +++ cat /tmp/tmp.CQxHbpZeQF +++ rm /tmp/tmp.566m0GimJp /tmp/tmp.CQxHbpZeQF +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.recreate-4603 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ygtArXyXHu +++ mktemp ++ local LAST_ERR=/tmp/tmp.RmELlfgWWn ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ygtArXyXHu ++ cat /tmp/tmp.RmELlfgWWn ++ rm /tmp/tmp.ygtArXyXHu /tmp/tmp.RmELlfgWWn ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.LY3qpIIvrg ++ mktemp + local LAST_ERR=/tmp/tmp.tcMjj7gqMD + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.LY3qpIIvrg + cat /tmp/tmp.tcMjj7gqMD + rm /tmp/tmp.LY3qpIIvrg /tmp/tmp.tcMjj7gqMD + return 0 + sleep 10 + desc 'Quick pgbouncer switch off and switch on' + set +o xtrace ----------------------------------------------------------------------------------- Quick pgbouncer switch off and switch on ----------------------------------------------------------------------------------- + disable_pgBouncer some-name + kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]' ++ mktemp + local LAST_OUT=/tmp/tmp.37ODnmgaJA ++ mktemp + local LAST_ERR=/tmp/tmp.wlCjhEMkxl + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.37ODnmgaJA perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.wlCjhEMkxl + rm /tmp/tmp.37ODnmgaJA /tmp/tmp.wlCjhEMkxl + return 0 + kubectl_bin wait --timeout=300s --for=delete deployment/some-name-pgbouncer ++ mktemp + local LAST_OUT=/tmp/tmp.L3Fx3jMcjd ++ mktemp + local LAST_ERR=/tmp/tmp.cGQDzhO2nW + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl wait --timeout=300s --for=delete deployment/some-name-pgbouncer + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.L3Fx3jMcjd deployment.apps/some-name-pgbouncer condition met + cat /tmp/tmp.cGQDzhO2nW + rm /tmp/tmp.L3Fx3jMcjd /tmp/tmp.cGQDzhO2nW + return 0 + enable_pgBouncer some-name 3 + local cluster_name=some-name + local replicas_num=3 + kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":3}]' ++ mktemp + local LAST_OUT=/tmp/tmp.46B4dyBtFj ++ mktemp + local LAST_ERR=/tmp/tmp.drEMcqCIIl + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":3}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.46B4dyBtFj perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.drEMcqCIIl + rm /tmp/tmp.46B4dyBtFj /tmp/tmp.drEMcqCIIl + return 0 + sleep 60 + kubectl_bin wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer ++ mktemp + local LAST_OUT=/tmp/tmp.1wOVlBev2E ++ mktemp + local LAST_ERR=/tmp/tmp.Kygz1OJKUi + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.1wOVlBev2E deployment.apps/some-name-pgbouncer condition met + cat /tmp/tmp.Kygz1OJKUi + rm /tmp/tmp.1wOVlBev2E /tmp/tmp.Kygz1OJKUi + return 0 + wait_deployment some-name-pgbouncer + local name=some-name-pgbouncer + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-pgbouncer ++ get_psql_user_pass pgbouncer some-name ++ local user=pgbouncer ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.L85j1zNjv6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2Pe6ssVuNR +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.L85j1zNjv6 +++ cat /tmp/tmp.2Pe6ssVuNR +++ rm /tmp/tmp.L85j1zNjv6 /tmp/tmp.2Pe6ssVuNR +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4Hr07Nmq7K ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0xfpamFf8M +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.4Hr07Nmq7K +++ cat /tmp/tmp.0xfpamFf8M +++ rm /tmp/tmp.4Hr07Nmq7K /tmp/tmp.0xfpamFf8M +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kEYcOU1pCr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RgKQdNZpaP +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.kEYcOU1pCr +++ cat /tmp/tmp.RgKQdNZpaP +++ rm /tmp/tmp.kEYcOU1pCr /tmp/tmp.RgKQdNZpaP +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.j2dIBBqI34 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.MdvJkoziw1 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.j2dIBBqI34 ++++ cat /tmp/tmp.MdvJkoziw1 ++++ rm /tmp/tmp.j2dIBBqI34 /tmp/tmp.MdvJkoziw1 ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.pgbouncer}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VDVEi35CvW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xCfOlae53k +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.pgbouncer}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.VDVEi35CvW +++ cat /tmp/tmp.xCfOlae53k +++ rm /tmp/tmp.VDVEi35CvW /tmp/tmp.xCfOlae53k +++ return 0 ++ echo pgbouncer_password + compare_psql_cmd show-1 'SHOW DATABASES' pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-4603 + local command_id=show-1 + local 'command=SHOW DATABASES' + local uri=pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-4603 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/show-1.sql + run_psql 'SHOW DATABASES' pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-4603 postgres '' + local 'command=SHOW DATABASES' + local uri=pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LaxNUHE4nb +++ mktemp ++ local LAST_ERR=/tmp/tmp.dVkny40GCJ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.LaxNUHE4nb ++ cat /tmp/tmp.dVkny40GCJ ++ rm /tmp/tmp.LaxNUHE4nb /tmp/tmp.dVkny40GCJ ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SHOW DATABASES\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.GH731gj6Pn ++ mktemp + local LAST_ERR=/tmp/tmp.FNcXSejYj3 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SHOW DATABASES\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.GH731gj6Pn + cat /tmp/tmp.FNcXSejYj3 + rm /tmp/tmp.GH731gj6Pn /tmp/tmp.FNcXSejYj3 + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/show-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/show-1.sql /tmp/tmp.UXRFYGRGBI/show-1.sql ++ get_psql_user_pass some-name some-name ++ local user=some-name ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.g0CiJ0wpmK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AYlAfIQZsW +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.g0CiJ0wpmK +++ cat /tmp/tmp.AYlAfIQZsW +++ rm /tmp/tmp.g0CiJ0wpmK /tmp/tmp.AYlAfIQZsW +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.waGkh82SSL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JMww5729pO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.waGkh82SSL +++ cat /tmp/tmp.JMww5729pO +++ rm /tmp/tmp.waGkh82SSL /tmp/tmp.JMww5729pO +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WpWAorsyyS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iqAO97jr9w +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.WpWAorsyyS +++ cat /tmp/tmp.iqAO97jr9w +++ rm /tmp/tmp.WpWAorsyyS /tmp/tmp.iqAO97jr9w +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.j3IcoU2Xkt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.vZKNIsog0W ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.j3IcoU2Xkt ++++ cat /tmp/tmp.vZKNIsog0W ++++ rm /tmp/tmp.j3IcoU2Xkt /tmp/tmp.vZKNIsog0W ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.s065Xh2o4W ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7Icjeejkrc +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.s065Xh2o4W +++ cat /tmp/tmp.7Icjeejkrc +++ rm /tmp/tmp.s065Xh2o4W /tmp/tmp.7Icjeejkrc +++ return 0 ++ echo some-name_pass + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.recreate-4603 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.recreate-4603 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.recreate-4603 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zm3hDTw6op +++ mktemp ++ local LAST_ERR=/tmp/tmp.v6Vuldcl8l ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.zm3hDTw6op ++ cat /tmp/tmp.v6Vuldcl8l ++ rm /tmp/tmp.zm3hDTw6op /tmp/tmp.v6Vuldcl8l ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.8lD0OwrcbB ++ mktemp + local LAST_ERR=/tmp/tmp.kTWtBt2IkP + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8lD0OwrcbB + cat /tmp/tmp.kTWtBt2IkP + rm /tmp/tmp.8lD0OwrcbB /tmp/tmp.kTWtBt2IkP + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.UXRFYGRGBI/select-1.sql + compare_kubectl deployment/some-name-pgbouncer + local resource=deployment/some-name-pgbouncer + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/deployment_some-name-pgbouncer.yml + local new_result=/tmp/tmp.UXRFYGRGBI/deployment_some-name-pgbouncer.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin get -o yaml deployment/some-name-pgbouncer + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.uid' + yq d - metadata.selfLink + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' ++ mktemp + local LAST_OUT=/tmp/tmp.Yf1twD28yR + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - metadata.deletionTimestamp + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**.creationTimestamp' ++ mktemp + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - spec.nodeName + local LAST_ERR=/tmp/tmp.XL8jIy7Bwx + local exit_status=0 + local timeout=4 + set +o errexit + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - '**.clusterIPs' + yq d - '**.volumeName' + yq d - '**.storageClassName' + yq d - '**.clusterIP' + yq d - '**.finalizers' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - spec.volumeMode + yq d - '**.nodePort' + yq d - '**.procMount' + yq d - '**.enableServiceLinks' + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**."percona.com/*"' + yq d - '**.controller-uid' + yq d - '**.healthCheckNodePort' + yq d - '**."kubernetes.io/pvc-protection"' + yq d - '**.internalTrafficPolicy' + yq d - '**.imagePullSecrets' + yq d - status ++ seq 0 2 + yq d - '**.dataSource' + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get -o yaml deployment/some-name-pgbouncer + yq d - '**.(name==suffix)' + yq d - '**.(name==PODNAME)' + yq d - '**.(name==NAMESPACE)' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - '**.image' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.preemptionPolicy' + yq d - '**.ipFamilies' + yq d - '**.ipFamilyPolicy' + yq d - '**.allocateLoadBalancerNodePorts' + yq d - metadata.resourceVersion + yq d - '**.namespace' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Yf1twD28yR + cat /tmp/tmp.XL8jIy7Bwx + rm /tmp/tmp.Yf1twD28yR /tmp/tmp.XL8jIy7Bwx + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.UXRFYGRGBI/deployment_some-name-pgbouncer.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/deployment_some-name-pgbouncer.yml /tmp/tmp.UXRFYGRGBI/deployment_some-name-pgbouncer.yml + compare_kubectl service/some-name-pgbouncer + local resource=service/some-name-pgbouncer + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/service_some-name-pgbouncer.yml + local new_result=/tmp/tmp.UXRFYGRGBI/service_some-name-pgbouncer.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.namespace' + yq d - '**.uid' + yq d - metadata.selfLink + yq d - metadata.deletionTimestamp + yq d - '**.image' + yq d - metadata.resourceVersion + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - '**.storageClassName' + yq d - '**.clusterIP' + yq d - '**.volumeName' + yq d - '**.clusterIPs' + yq d - '**.dataSource' + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - '**.creationTimestamp' + yq d - '**.procMount' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - '**.finalizers' + yq d - spec.volumeMode + yq d - '**."kubernetes.io/pvc-protection"' + yq d - spec.nodeName + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - '**.imagePullSecrets' + yq d - '**.nodePort' + yq d - '**.enableServiceLinks' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - '**.(name==NAMESPACE)' + yq d - '**.controller-uid' + yq d - '**."percona.com/*"' + yq d - '**.ipFamilies' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**.internalTrafficPolicy' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - '**.healthCheckNodePort' + yq d - status + yq d - '**.preemptionPolicy' + yq d - '**.(name==suffix)' + yq d - '**.ipFamilyPolicy' + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**.(name==PODNAME)' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**.allocateLoadBalancerNodePorts' + kubectl_bin get -o yaml service/some-name-pgbouncer ++ mktemp + local LAST_OUT=/tmp/tmp.QRfwqscIAm ++ mktemp + local LAST_ERR=/tmp/tmp.bA3vvUJNYP + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get -o yaml service/some-name-pgbouncer + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.QRfwqscIAm + cat /tmp/tmp.bA3vvUJNYP + rm /tmp/tmp.QRfwqscIAm /tmp/tmp.bA3vvUJNYP + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.UXRFYGRGBI/service_some-name-pgbouncer.yml kind ++ yq r /tmp/tmp.UXRFYGRGBI/service_some-name-pgbouncer.yml metadata.annotations + [[ x{} == \x ]] ++ yq r /tmp/tmp.UXRFYGRGBI/service_some-name-pgbouncer.yml metadata.annotations + [[ x{} == \x\{\} ]] + yq d -i /tmp/tmp.UXRFYGRGBI/service_some-name-pgbouncer.yml metadata.annotations + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/service_some-name-pgbouncer.yml /tmp/tmp.UXRFYGRGBI/service_some-name-pgbouncer.yml ++ seq 1 3 + for repl in '$(seq 1 3)' + desc 'check replica 1' + set +o xtrace ----------------------------------------------------------------------------------- check replica 1 ----------------------------------------------------------------------------------- + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' ++ mktemp + local LAST_OUT=/tmp/tmp.S6BVMbQZjN ++ mktemp + local LAST_ERR=/tmp/tmp.Po6XxS8hFe + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.S6BVMbQZjN perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.Po6XxS8hFe + rm /tmp/tmp.S6BVMbQZjN /tmp/tmp.Po6XxS8hFe + return 0 + sleep 30 + wait_deployment some-name-repl1 + local name=some-name-repl1 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl1 + sleep 30 + check_replica some-name-repl1 some-name select-1 + local name=some-name-repl1 + local cluster=some-name + local sql_cmp=select-1 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hKLk9cMBDT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mwKgcPWSmq +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.hKLk9cMBDT +++ cat /tmp/tmp.mwKgcPWSmq +++ rm /tmp/tmp.hKLk9cMBDT /tmp/tmp.mwKgcPWSmq +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6rlfoEeb4g ++++ mktemp +++ local LAST_ERR=/tmp/tmp.QgOj058PTY +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.6rlfoEeb4g +++ cat /tmp/tmp.QgOj058PTY +++ rm /tmp/tmp.6rlfoEeb4g /tmp/tmp.QgOj058PTY +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sfrLziArPW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xsKD0ZmghW +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.sfrLziArPW +++ cat /tmp/tmp.xsKD0ZmghW +++ rm /tmp/tmp.sfrLziArPW /tmp/tmp.xsKD0ZmghW +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Ckvh6UMcXh +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.jWfG7BAVMm ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.Ckvh6UMcXh ++++ cat /tmp/tmp.jWfG7BAVMm ++++ rm /tmp/tmp.Ckvh6UMcXh /tmp/tmp.jWfG7BAVMm ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.f23YP6wKRo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Trxys0Ak0h +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.f23YP6wKRo +++ cat /tmp/tmp.Trxys0Ak0h +++ rm /tmp/tmp.f23YP6wKRo /tmp/tmp.Trxys0Ak0h +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-4603 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VcmvDUrbrK +++ mktemp ++ local LAST_ERR=/tmp/tmp.r3el6aKnyy ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.VcmvDUrbrK ++ cat /tmp/tmp.r3el6aKnyy ++ rm /tmp/tmp.VcmvDUrbrK /tmp/tmp.r3el6aKnyy ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.3WhOIxy7S8 ++ mktemp + local LAST_ERR=/tmp/tmp.AXkQsk3Vxs + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.3WhOIxy7S8 + cat /tmp/tmp.AXkQsk3Vxs + rm /tmp/tmp.3WhOIxy7S8 /tmp/tmp.AXkQsk3Vxs + return 0 ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XIQUIG8CX4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.YFVANfK5SR ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.XIQUIG8CX4 ++ cat /tmp/tmp.YFVANfK5SR ++ rm /tmp/tmp.XIQUIG8CX4 /tmp/tmp.YFVANfK5SR ++ return 0 + replica_pod_name=some-name-repl1-7987df9b7-8wj65 ++ grep some-name-repl1-7987df9b7-8wj65 /tmp/tmp.UXRFYGRGBI/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl1-7987df9b7-8wj65 | 10.109.89.50 | streaming ]] ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qHSd6N0Ai3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Snmov9ixdd ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.qHSd6N0Ai3 ++ cat /tmp/tmp.Snmov9ixdd ++ rm /tmp/tmp.qHSd6N0Ai3 /tmp/tmp.Snmov9ixdd ++ return 0 + replica_pod_IP=10.109.89.50 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.89.50 '' ' ' + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.89.50 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.89.50 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.89.50 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9tmZnbf5Ik +++ mktemp ++ local LAST_ERR=/tmp/tmp.f95XOl7ylI ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.9tmZnbf5Ik ++ cat /tmp/tmp.f95XOl7ylI ++ rm /tmp/tmp.9tmZnbf5Ik /tmp/tmp.f95XOl7ylI ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.89.50 ' ++ mktemp + local LAST_OUT=/tmp/tmp.meekAOZOqM ++ mktemp + local LAST_ERR=/tmp/tmp.GrNaVAwwR8 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.89.50 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.meekAOZOqM + cat /tmp/tmp.GrNaVAwwR8 + rm /tmp/tmp.meekAOZOqM /tmp/tmp.GrNaVAwwR8 + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.UXRFYGRGBI/select-1.sql + for repl in '$(seq 1 3)' + desc 'check replica 2' + set +o xtrace ----------------------------------------------------------------------------------- check replica 2 ----------------------------------------------------------------------------------- + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' ++ mktemp + local LAST_OUT=/tmp/tmp.upZQIEonxQ ++ mktemp + local LAST_ERR=/tmp/tmp.2PPA6eEACQ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.upZQIEonxQ perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.2PPA6eEACQ + rm /tmp/tmp.upZQIEonxQ /tmp/tmp.2PPA6eEACQ + return 0 + sleep 30 + wait_deployment some-name-repl2 + local name=some-name-repl2 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl2 + sleep 30 + check_replica some-name-repl2 some-name select-1 + local name=some-name-repl2 + local cluster=some-name + local sql_cmp=select-1 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QECsgh08yl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XKABkuHHrU +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.QECsgh08yl +++ cat /tmp/tmp.XKABkuHHrU +++ rm /tmp/tmp.QECsgh08yl /tmp/tmp.XKABkuHHrU +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vBETg8esSw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.j40dp62OC7 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.vBETg8esSw +++ cat /tmp/tmp.j40dp62OC7 +++ rm /tmp/tmp.vBETg8esSw /tmp/tmp.j40dp62OC7 +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IhDntUvmmY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gGY28UxlaH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.IhDntUvmmY +++ cat /tmp/tmp.gGY28UxlaH +++ rm /tmp/tmp.IhDntUvmmY /tmp/tmp.gGY28UxlaH +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ILTkuSc99x +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.RsM5lAgE8b ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ILTkuSc99x ++++ cat /tmp/tmp.RsM5lAgE8b ++++ rm /tmp/tmp.ILTkuSc99x /tmp/tmp.RsM5lAgE8b ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gRvbze42bS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.V8UBOGNcJH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.gRvbze42bS +++ cat /tmp/tmp.V8UBOGNcJH +++ rm /tmp/tmp.gRvbze42bS /tmp/tmp.V8UBOGNcJH +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-4603 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qe9zIOsdZ9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gQuy0Bj2MO ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Qe9zIOsdZ9 ++ cat /tmp/tmp.gQuy0Bj2MO ++ rm /tmp/tmp.Qe9zIOsdZ9 /tmp/tmp.gQuy0Bj2MO ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.ZtU95ORNoi ++ mktemp + local LAST_ERR=/tmp/tmp.rChauimReU + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ZtU95ORNoi + cat /tmp/tmp.rChauimReU + rm /tmp/tmp.ZtU95ORNoi /tmp/tmp.rChauimReU + return 0 ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.t5ePnfgIia +++ mktemp ++ local LAST_ERR=/tmp/tmp.wMFRrLBPM1 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.t5ePnfgIia ++ cat /tmp/tmp.wMFRrLBPM1 ++ rm /tmp/tmp.t5ePnfgIia /tmp/tmp.wMFRrLBPM1 ++ return 0 + replica_pod_name=some-name-repl2-57cf7d6dcf-4jjhb ++ grep some-name-repl2-57cf7d6dcf-4jjhb /tmp/tmp.UXRFYGRGBI/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl2-57cf7d6dcf-4jjhb | 10.109.88.57 | streaming ]] ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yBxtLIfSHb +++ mktemp ++ local LAST_ERR=/tmp/tmp.R415ZEKade ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.yBxtLIfSHb ++ cat /tmp/tmp.R415ZEKade ++ rm /tmp/tmp.yBxtLIfSHb /tmp/tmp.R415ZEKade ++ return 0 + replica_pod_IP=10.109.88.57 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.88.57 '' ' ' + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.88.57 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.88.57 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.88.57 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Xwwu374s8e +++ mktemp ++ local LAST_ERR=/tmp/tmp.ybQ4IzDPnc ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Xwwu374s8e ++ cat /tmp/tmp.ybQ4IzDPnc ++ rm /tmp/tmp.Xwwu374s8e /tmp/tmp.ybQ4IzDPnc ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.88.57 ' ++ mktemp + local LAST_OUT=/tmp/tmp.AYjSTmJFpp ++ mktemp + local LAST_ERR=/tmp/tmp.qf0PxByV9P + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.88.57 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.AYjSTmJFpp + cat /tmp/tmp.qf0PxByV9P + rm /tmp/tmp.AYjSTmJFpp /tmp/tmp.qf0PxByV9P + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.UXRFYGRGBI/select-1.sql + for repl in '$(seq 1 3)' + desc 'check replica 3' + set +o xtrace ----------------------------------------------------------------------------------- check replica 3 ----------------------------------------------------------------------------------- + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]' ++ mktemp + local LAST_OUT=/tmp/tmp.kyzFx4H9Dj ++ mktemp + local LAST_ERR=/tmp/tmp.PRUJDwKEWL + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.kyzFx4H9Dj perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.PRUJDwKEWL + rm /tmp/tmp.kyzFx4H9Dj /tmp/tmp.PRUJDwKEWL + return 0 + sleep 30 + wait_deployment some-name-repl3 + local name=some-name-repl3 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl3 + sleep 30 + check_replica some-name-repl3 some-name select-1 + local name=some-name-repl3 + local cluster=some-name + local sql_cmp=select-1 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ogeZQ1bYjX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FxapD5BYs6 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ogeZQ1bYjX +++ cat /tmp/tmp.FxapD5BYs6 +++ rm /tmp/tmp.ogeZQ1bYjX /tmp/tmp.FxapD5BYs6 +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zc7zENgI7T ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ImzxABv6RW +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zc7zENgI7T +++ cat /tmp/tmp.ImzxABv6RW +++ rm /tmp/tmp.zc7zENgI7T /tmp/tmp.ImzxABv6RW +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.naBEFess2E ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uvtTx1iXZ2 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.naBEFess2E +++ cat /tmp/tmp.uvtTx1iXZ2 +++ rm /tmp/tmp.naBEFess2E /tmp/tmp.uvtTx1iXZ2 +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Am49aBdHYP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.MhSa41PRgb ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.Am49aBdHYP ++++ cat /tmp/tmp.MhSa41PRgb ++++ rm /tmp/tmp.Am49aBdHYP /tmp/tmp.MhSa41PRgb ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3ldpQXtVMI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2lF7ynXz2u +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.3ldpQXtVMI +++ cat /tmp/tmp.2lF7ynXz2u +++ rm /tmp/tmp.3ldpQXtVMI /tmp/tmp.2lF7ynXz2u +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-4603 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iO7hcckndr +++ mktemp ++ local LAST_ERR=/tmp/tmp.apEti6Zv0D ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.iO7hcckndr ++ cat /tmp/tmp.apEti6Zv0D ++ rm /tmp/tmp.iO7hcckndr /tmp/tmp.apEti6Zv0D ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.gxTwD1pc4s ++ mktemp + local LAST_ERR=/tmp/tmp.qxE4hC7ALS + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.gxTwD1pc4s + cat /tmp/tmp.qxE4hC7ALS + rm /tmp/tmp.gxTwD1pc4s /tmp/tmp.qxE4hC7ALS + return 0 ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jrwJuLjfTM +++ mktemp ++ local LAST_ERR=/tmp/tmp.NHklVGvmDK ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.jrwJuLjfTM ++ cat /tmp/tmp.NHklVGvmDK ++ rm /tmp/tmp.jrwJuLjfTM /tmp/tmp.NHklVGvmDK ++ return 0 + replica_pod_name=some-name-repl3-97b8b7b55-6hvw9 ++ grep some-name-repl3-97b8b7b55-6hvw9 /tmp/tmp.UXRFYGRGBI/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl3-97b8b7b55-6hvw9 | 10.109.90.33 | streaming ]] ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kpftaifkpy +++ mktemp ++ local LAST_ERR=/tmp/tmp.8efNcw6q68 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.kpftaifkpy ++ cat /tmp/tmp.8efNcw6q68 ++ rm /tmp/tmp.kpftaifkpy /tmp/tmp.8efNcw6q68 ++ return 0 + replica_pod_IP=10.109.90.33 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.90.33 '' ' ' + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.90.33 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.90.33 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.90.33 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nfMg3lKAjJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.rr6qvYmwTv ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.nfMg3lKAjJ ++ cat /tmp/tmp.rr6qvYmwTv ++ rm /tmp/tmp.nfMg3lKAjJ /tmp/tmp.rr6qvYmwTv ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.90.33 ' ++ mktemp + local LAST_OUT=/tmp/tmp.8086k2TTRD ++ mktemp + local LAST_ERR=/tmp/tmp.NOZQHURG78 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.90.33 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8086k2TTRD + cat /tmp/tmp.NOZQHURG78 + rm /tmp/tmp.8086k2TTRD /tmp/tmp.NOZQHURG78 + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.UXRFYGRGBI/select-1.sql + desc 'Restart the cluster' + set +o xtrace ----------------------------------------------------------------------------------- Restart the cluster ----------------------------------------------------------------------------------- + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepData","value":true}]' ++ mktemp + local LAST_OUT=/tmp/tmp.RZxmPjSe5x ++ mktemp + local LAST_ERR=/tmp/tmp.8kfow2fcQ6 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepData","value":true}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.RZxmPjSe5x perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.8kfow2fcQ6 + rm /tmp/tmp.RZxmPjSe5x /tmp/tmp.8kfow2fcQ6 + return 0 + sleep 30 ++ seq 1 3 + for i in '$(seq 1 3)' + sleep 15 + wait_deployment some-name-repl1 + local name=some-name-repl1 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl1............................................ + for i in '$(seq 1 3)' + sleep 15 + wait_deployment some-name-repl2 + local name=some-name-repl2 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl2 + for i in '$(seq 1 3)' + sleep 15 + wait_deployment some-name-repl3 + local name=some-name-repl3 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl3....... + wait_deployment some-name + local name=some-name + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name....... + sleep 30 + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' ++ mktemp + local LAST_OUT=/tmp/tmp.AKNGrlQU7A ++ mktemp + local LAST_ERR=/tmp/tmp.ZzBW2NKYOy + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.AKNGrlQU7A perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.ZzBW2NKYOy + rm /tmp/tmp.AKNGrlQU7A /tmp/tmp.ZzBW2NKYOy + return 0 + wait_for_delete pgreplica/some-name-repl3 + local res=pgreplica/some-name-repl3 + set +o xtrace pgreplica/some-name-repl3 - ....Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl3" not found + sleep 30 + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' ++ mktemp + local LAST_OUT=/tmp/tmp.2kI3qO62NT ++ mktemp + local LAST_ERR=/tmp/tmp.6CkooOjYYE + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2kI3qO62NT perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.6CkooOjYYE + rm /tmp/tmp.2kI3qO62NT /tmp/tmp.6CkooOjYYE + return 0 + wait_for_delete pgreplica/some-name-repl2 + local res=pgreplica/some-name-repl2 + set +o xtrace pgreplica/some-name-repl2 - ....Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl2" not found + sleep 30 + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 0}]' ++ mktemp + local LAST_OUT=/tmp/tmp.v2KNEQIGNb ++ mktemp + local LAST_ERR=/tmp/tmp.70jMoKUWD1 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 0}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.v2KNEQIGNb perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.70jMoKUWD1 + rm /tmp/tmp.v2KNEQIGNb /tmp/tmp.70jMoKUWD1 + return 0 + wait_for_delete pgreplica/some-name-repl1 + local res=pgreplica/some-name-repl1 + set +o xtrace pgreplica/some-name-repl1 - ....Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl1" not found + sleep 30 + kubectl_bin delete perconapgclusters --all ++ mktemp + local LAST_OUT=/tmp/tmp.LbAJIQZJKW ++ mktemp + local LAST_ERR=/tmp/tmp.FxIe9pJOJN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete perconapgclusters --all + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.LbAJIQZJKW perconapgcluster.pg.percona.com "some-name" deleted + cat /tmp/tmp.FxIe9pJOJN + rm /tmp/tmp.LbAJIQZJKW /tmp/tmp.FxIe9pJOJN + return 0 + wait_for_delete pgcluster/some-name + local res=pgcluster/some-name + set +o xtrace pgcluster/some-name - Error from server (NotFound): pgclusters.pg.percona.com "some-name" not found + wait_for_delete deployment/some-name + local res=deployment/some-name + set +o xtrace deployment/some-name - Error from server (NotFound): deployments.apps "some-name" not found + wait_for_delete deployment/some-name-backrest-shared-repo + local res=deployment/some-name-backrest-shared-repo + set +o xtrace deployment/some-name-backrest-shared-repo - Error from server (NotFound): deployments.apps "some-name-backrest-shared-repo" not found + compare_kubectl pvc/some-name + local resource=pvc/some-name + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/pvc_some-name.yml + local new_result=/tmp/tmp.UXRFYGRGBI/pvc_some-name.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin get -o yaml pvc/some-name + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.namespace' ++ mktemp + local LAST_OUT=/tmp/tmp.F909GZDcx1 + yq d - metadata.resourceVersion + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - 'metadata.annotations."kubernetes.io/psp"' ++ mktemp + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - metadata.selfLink + local LAST_ERR=/tmp/tmp.blmvymlkvu + local exit_status=0 + local timeout=4 + set +o errexit + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**.procMount' + yq d - '**.image' + yq d - '**.creationTimestamp' + yq d - '**.finalizers' + yq d - '**.clusterIPs' + yq d - '**.dataSource' + yq d - '**.clusterIP' ++ seq 0 2 + yq d - '**."percona.com/*"' + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - spec.volumeMode + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - spec.nodeName + for i in '$(seq 0 2)' + yq d - '**.imagePullSecrets' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get -o yaml pvc/some-name + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**.storageClassName' + yq d - '**.enableServiceLinks' + yq d - '**.volumeName' + yq d - status + yq d - '**.(name==suffix)' + yq d - '**.nodePort' + yq d - '**.healthCheckNodePort' + yq d - '**."kubernetes.io/pvc-protection"' + yq d - metadata.deletionTimestamp + yq d - '**.(name==COMMAND_OPTS)' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.(name==PODNAME)' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - '**.(name==NAMESPACE)' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**.controller-uid' + yq d - '**.preemptionPolicy' + yq d - '**.ipFamilies' + yq d - '**.ipFamilyPolicy' + yq d - '**.internalTrafficPolicy' + yq d - '**.allocateLoadBalancerNodePorts' + yq d - '**.uid' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.F909GZDcx1 + cat /tmp/tmp.blmvymlkvu + rm /tmp/tmp.F909GZDcx1 /tmp/tmp.blmvymlkvu + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.UXRFYGRGBI/pvc_some-name.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/pvc_some-name.yml /tmp/tmp.UXRFYGRGBI/pvc_some-name.yml + compare_kubectl pvc/some-name-repl1 + local resource=pvc/some-name-repl1 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/pvc_some-name-repl1.yml + local new_result=/tmp/tmp.UXRFYGRGBI/pvc_some-name-repl1.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.namespace' + yq d - '**.uid' + yq d - metadata.selfLink + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - metadata.resourceVersion + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - '**.creationTimestamp' + yq d - '**.image' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - '**.clusterIP' + yq d - '**.clusterIPs' + yq d - metadata.deletionTimestamp + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - '**.procMount' + yq d - '**.dataSource' + yq d - '**.finalizers' + yq d - '**.storageClassName' + yq d - '**.imagePullSecrets' + yq d - '**."kubernetes.io/pvc-protection"' + yq d - '**.volumeName' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - spec.volumeMode + yq d - '**.enableServiceLinks' + yq d - spec.nodeName + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - status + yq d - '**.nodePort' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - '**."percona.com/*"' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**.controller-uid' + yq d - '**.ipFamilies' + kubectl_bin get -o yaml pvc/some-name-repl1 + yq d - '**.preemptionPolicy' + yq d - '**.healthCheckNodePort' + yq d - '**.(name==suffix)' + yq d - '**.(name==NAMESPACE)' + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**.(name==PODNAME)' + yq d - '**.ipFamilyPolicy' ++ mktemp + yq d - '**.internalTrafficPolicy' + yq d - '**.allocateLoadBalancerNodePorts' + local LAST_OUT=/tmp/tmp.l1k6yZkxDt ++ mktemp + local LAST_ERR=/tmp/tmp.QEsgp7zr23 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get -o yaml pvc/some-name-repl1 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.l1k6yZkxDt + cat /tmp/tmp.QEsgp7zr23 + rm /tmp/tmp.l1k6yZkxDt /tmp/tmp.QEsgp7zr23 + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.UXRFYGRGBI/pvc_some-name-repl1.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/pvc_some-name-repl1.yml /tmp/tmp.UXRFYGRGBI/pvc_some-name-repl1.yml + compare_kubectl pvc/some-name-repl2 + local resource=pvc/some-name-repl2 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/pvc_some-name-repl2.yml + local new_result=/tmp/tmp.UXRFYGRGBI/pvc_some-name-repl2.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.namespace' + yq d - '**.uid' + yq d - metadata.resourceVersion + yq d - metadata.selfLink + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - metadata.deletionTimestamp + yq d - '**.clusterIP' + yq d - '**.image' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - '**.clusterIPs' + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - '**.(name==COMMAND_OPTS)' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**.(name==suffix)' + yq d - '**.creationTimestamp' + yq d - '**.ipFamilies' + yq d - '**.enableServiceLinks' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**.dataSource' + yq d - '**.imagePullSecrets' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**.procMount' + yq d - '**.healthCheckNodePort' + yq d - '**.(name==PODNAME)' + yq d - '**.storageClassName' + yq d - '**.(name==NAMESPACE)' + yq d - '**.ipFamilyPolicy' + yq d - '**.finalizers' + yq d - status + yq d - '**."kubernetes.io/pvc-protection"' + yq d - '**.nodePort' + yq d - '**.volumeName' + yq d - '**."percona.com/*"' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - spec.volumeMode + yq d - spec.nodeName + yq d - '**.controller-uid' + yq d - '**.preemptionPolicy' + yq d - '**.internalTrafficPolicy' + yq d - '**.allocateLoadBalancerNodePorts' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + kubectl_bin get -o yaml pvc/some-name-repl2 ++ mktemp + local LAST_OUT=/tmp/tmp.a8NIvtGGoq ++ mktemp + local LAST_ERR=/tmp/tmp.PyPHP5pNAb + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get -o yaml pvc/some-name-repl2 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.a8NIvtGGoq + cat /tmp/tmp.PyPHP5pNAb + rm /tmp/tmp.a8NIvtGGoq /tmp/tmp.PyPHP5pNAb + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.UXRFYGRGBI/pvc_some-name-repl2.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/pvc_some-name-repl2.yml /tmp/tmp.UXRFYGRGBI/pvc_some-name-repl2.yml + compare_kubectl pvc/some-name-repl3 + local resource=pvc/some-name-repl3 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/pvc_some-name-repl3.yml + local new_result=/tmp/tmp.UXRFYGRGBI/pvc_some-name-repl3.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.namespace' + yq d - '**.uid' + yq d - metadata.deletionTimestamp + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - metadata.resourceVersion + yq d - metadata.selfLink + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - '**.creationTimestamp' + yq d - '**.image' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**.clusterIP' + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - '**.clusterIPs' + yq d - '**.procMount' + yq d - '**.dataSource' + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - '**.storageClassName' + yq d - '**.volumeName' + yq d - '**.finalizers' + yq d - '**."kubernetes.io/pvc-protection"' + yq d - spec.volumeMode + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - '**.(name==NAMESPACE)' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - spec.nodeName + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - '**."percona.com/*"' + yq d - '**.(name==PODNAME)' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**.ipFamilies' + yq d - '**.ipFamilyPolicy' + yq d - '**.healthCheckNodePort' + yq d - '**.internalTrafficPolicy' + yq d - '**.nodePort' + yq d - '**.imagePullSecrets' + yq d - '**.enableServiceLinks' + yq d - status + yq d - '**.(name==suffix)' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.controller-uid' + yq d - '**.preemptionPolicy' + yq d - '**.allocateLoadBalancerNodePorts' + kubectl_bin get -o yaml pvc/some-name-repl3 ++ mktemp + local LAST_OUT=/tmp/tmp.L7pjwMQ8Y4 ++ mktemp + local LAST_ERR=/tmp/tmp.zITm58LROb + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get -o yaml pvc/some-name-repl3 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.L7pjwMQ8Y4 + cat /tmp/tmp.zITm58LROb + rm /tmp/tmp.L7pjwMQ8Y4 /tmp/tmp.zITm58LROb + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.UXRFYGRGBI/pvc_some-name-repl3.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/pvc_some-name-repl3.yml /tmp/tmp.UXRFYGRGBI/pvc_some-name-repl3.yml + sleep 30 + create_user_secrets some-name + local cluster=some-name + local namespace=recreate-4603 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/users-secret.yaml + yq w - metadata.name some-name-users-secret + yq w - stringData.some-name some-name_pass + kubectl_bin -n recreate-4603 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.9Axr7FXyWk ++ mktemp + local LAST_ERR=/tmp/tmp.TP2a9QjG8h + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.9Axr7FXyWk secret/some-name-users-secret configured + cat /tmp/tmp.TP2a9QjG8h + rm /tmp/tmp.9Axr7FXyWk /tmp/tmp.TP2a9QjG8h + return 0 + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/users-secret.yaml + yq w - metadata.name some-name-pmm-secret + kubectl_bin -n recreate-4603 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ntOGSV8rR1 ++ mktemp + local LAST_ERR=/tmp/tmp.L9fq66Yqv2 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ntOGSV8rR1 secret/some-name-pmm-secret configured + cat /tmp/tmp.L9fq66Yqv2 + rm /tmp/tmp.ntOGSV8rR1 /tmp/tmp.L9fq66Yqv2 + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/cr.yaml some-name + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/cr.yaml + local name=some-name + local backup=false + local restore_from=false + local restore_opts=false + local custom_config=false + local podAntiAffinity= + local schedule= + local schedule_type= + local tls_only= + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/cr.yaml metadata.annotations.current-primary some-name + yq w - metadata.labels.crunchy-pgha-scope some-name + yq w - metadata.labels.deployment-name some-name + yq w - metadata.labels.name some-name + yq w - spec.keepData false + yq w - metadata.labels.pg-cluster some-name + yq w - metadata.name some-name + yq w - spec.database some-name + yq w - spec.clustername some-name + yq w - spec.name some-name + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.secretsName some-name-users-secret + yq w - spec.pmm.image perconalab/pmm-client:2.41.0 + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.user some-name + yq w - spec.keepBackups false + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + case ${podAntiAffinity} in + case ${backup} in + [[ false != \f\a\l\s\e ]] + [[ false != \f\a\l\s\e ]] + [[ '' == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.UXRFYGRGBI/cr.yaml -n recreate-4603 ++ mktemp + local LAST_OUT=/tmp/tmp.3sVLnNhXud ++ mktemp + local LAST_ERR=/tmp/tmp.BsJGbrRfUx + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl apply -f /tmp/tmp.UXRFYGRGBI/cr.yaml -n recreate-4603 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.3sVLnNhXud perconapgcluster.pg.percona.com/some-name created + cat /tmp/tmp.BsJGbrRfUx + rm /tmp/tmp.3sVLnNhXud /tmp/tmp.BsJGbrRfUx + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=recreate-4603 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hRhXvL0IUk +++ mktemp ++ local LAST_ERR=/tmp/tmp.g9yRmA2Vhc ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.hRhXvL0IUk ++ cat /tmp/tmp.g9yRmA2Vhc ++ rm /tmp/tmp.hRhXvL0IUk /tmp/tmp.g9yRmA2Vhc ++ return 0 + status_query='kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RB33YQsuFG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aKgS9sLhf5 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RB33YQsuFG +++ cat /tmp/tmp.aKgS9sLhf5 +++ rm /tmp/tmp.RB33YQsuFG /tmp/tmp.aKgS9sLhf5 +++ return 0 + [[ '' == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 1 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JanrTkEXrU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.c6ENEQ0pHj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.JanrTkEXrU +++ cat /tmp/tmp.c6ENEQ0pHj +++ rm /tmp/tmp.JanrTkEXrU /tmp/tmp.c6ENEQ0pHj +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 2 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.91KwdTDKRw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.854nmEDjYj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.91KwdTDKRw +++ cat /tmp/tmp.854nmEDjYj +++ rm /tmp/tmp.91KwdTDKRw /tmp/tmp.854nmEDjYj +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 3 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.27U9uocuyZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZW2GtxBhUW +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.27U9uocuyZ +++ cat /tmp/tmp.ZW2GtxBhUW +++ rm /tmp/tmp.27U9uocuyZ /tmp/tmp.ZW2GtxBhUW +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 4 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UMuHF67e0x ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TzoOmMxUYH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.UMuHF67e0x +++ cat /tmp/tmp.TzoOmMxUYH +++ rm /tmp/tmp.UMuHF67e0x /tmp/tmp.TzoOmMxUYH +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 5 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IcVrUO7hGB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ORJ7XzhXAY +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.IcVrUO7hGB +++ cat /tmp/tmp.ORJ7XzhXAY +++ rm /tmp/tmp.IcVrUO7hGB /tmp/tmp.ORJ7XzhXAY +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 6 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RCXqt2XHeA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VGLINNqXc1 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RCXqt2XHeA +++ cat /tmp/tmp.VGLINNqXc1 +++ rm /tmp/tmp.RCXqt2XHeA /tmp/tmp.VGLINNqXc1 +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 7 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9KGfuykyb0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sbMcLA0LXM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.9KGfuykyb0 +++ cat /tmp/tmp.sbMcLA0LXM +++ rm /tmp/tmp.9KGfuykyb0 /tmp/tmp.sbMcLA0LXM +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 20 ++ get_psql_user_pass some-name some-name ++ local user=some-name ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QoaKfrlJbI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JKIYjzbrcV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.QoaKfrlJbI +++ cat /tmp/tmp.JKIYjzbrcV +++ rm /tmp/tmp.QoaKfrlJbI /tmp/tmp.JKIYjzbrcV +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BU6zm4wibi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.16iM4rArnh +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.BU6zm4wibi +++ cat /tmp/tmp.16iM4rArnh +++ rm /tmp/tmp.BU6zm4wibi /tmp/tmp.16iM4rArnh +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8D1sMoUYkm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PYnHQi0HLO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.8D1sMoUYkm +++ cat /tmp/tmp.PYnHQi0HLO +++ rm /tmp/tmp.8D1sMoUYkm /tmp/tmp.PYnHQi0HLO +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.fcCsDYoS2T +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.WDhtXkfFRu ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.fcCsDYoS2T ++++ cat /tmp/tmp.WDhtXkfFRu ++++ rm /tmp/tmp.fcCsDYoS2T /tmp/tmp.WDhtXkfFRu ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AT3c5XCW2d ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6DXwuKPONO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.AT3c5XCW2d +++ cat /tmp/tmp.6DXwuKPONO +++ rm /tmp/tmp.AT3c5XCW2d /tmp/tmp.6DXwuKPONO +++ return 0 ++ echo some-name_pass + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name.recreate-4603 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name.recreate-4603 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name.recreate-4603 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d2Hy3lXkzC +++ mktemp ++ local LAST_ERR=/tmp/tmp.WMWcpsVpQt ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.d2Hy3lXkzC ++ cat /tmp/tmp.WMWcpsVpQt ++ rm /tmp/tmp.d2Hy3lXkzC /tmp/tmp.WMWcpsVpQt ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.AJ5GyLeVpQ ++ mktemp + local LAST_ERR=/tmp/tmp.FW4WL46Hef + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.AJ5GyLeVpQ + cat /tmp/tmp.FW4WL46Hef + rm /tmp/tmp.AJ5GyLeVpQ /tmp/tmp.FW4WL46Hef + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.UXRFYGRGBI/select-1.sql ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BgASJeD9co ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8oSu264BkF +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.BgASJeD9co +++ cat /tmp/tmp.8oSu264BkF +++ rm /tmp/tmp.BgASJeD9co /tmp/tmp.8oSu264BkF +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uPFos4DOpE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.L3mwGVshXq +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uPFos4DOpE +++ cat /tmp/tmp.L3mwGVshXq +++ rm /tmp/tmp.uPFos4DOpE /tmp/tmp.L3mwGVshXq +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eAOqagLZgc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Jj8J7jQUWl +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.eAOqagLZgc +++ cat /tmp/tmp.Jj8J7jQUWl +++ rm /tmp/tmp.eAOqagLZgc /tmp/tmp.Jj8J7jQUWl +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.KtJFv0gKw3 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.XiefO4238k ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.KtJFv0gKw3 ++++ cat /tmp/tmp.XiefO4238k ++++ rm /tmp/tmp.KtJFv0gKw3 /tmp/tmp.XiefO4238k ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CWv3HCxkGr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nCLwoQKdEb +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.CWv3HCxkGr +++ cat /tmp/tmp.nCLwoQKdEb +++ rm /tmp/tmp.CWv3HCxkGr /tmp/tmp.nCLwoQKdEb +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name.recreate-4603 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6LdZCZVtCk +++ mktemp ++ local LAST_ERR=/tmp/tmp.9Nj1slGNrK ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.6LdZCZVtCk ++ cat /tmp/tmp.9Nj1slGNrK ++ rm /tmp/tmp.6LdZCZVtCk /tmp/tmp.9Nj1slGNrK ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.9WCOYxxizI ++ mktemp + local LAST_ERR=/tmp/tmp.oherxIccuD + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.9WCOYxxizI + cat /tmp/tmp.oherxIccuD + rm /tmp/tmp.9WCOYxxizI /tmp/tmp.oherxIccuD + return 0 ++ seq 1 3 + for r in '$(seq 1 3)' + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' ++ mktemp + local LAST_OUT=/tmp/tmp.6ZOqPl24ZU ++ mktemp + local LAST_ERR=/tmp/tmp.CkItJbCrK2 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.6ZOqPl24ZU perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.CkItJbCrK2 + rm /tmp/tmp.6ZOqPl24ZU /tmp/tmp.CkItJbCrK2 + return 0 + sleep 25 + wait_deployment some-name-repl1 + local name=some-name-repl1 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl1 + sleep 35 + check_replica some-name-repl1 some-name select-2 + local name=some-name-repl1 + local cluster=some-name + local sql_cmp=select-2 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TiXQhSqKuq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.P3pvWU55wZ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TiXQhSqKuq +++ cat /tmp/tmp.P3pvWU55wZ +++ rm /tmp/tmp.TiXQhSqKuq /tmp/tmp.P3pvWU55wZ +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.U6boiik0ck ++++ mktemp +++ local LAST_ERR=/tmp/tmp.acG7ZkY0wX +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.U6boiik0ck +++ cat /tmp/tmp.acG7ZkY0wX +++ rm /tmp/tmp.U6boiik0ck /tmp/tmp.acG7ZkY0wX +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iLQeacdlaM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W9TlpugYdx +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.iLQeacdlaM +++ cat /tmp/tmp.W9TlpugYdx +++ rm /tmp/tmp.iLQeacdlaM /tmp/tmp.W9TlpugYdx +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.B5lwUVIb0e +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.EkJO1Nc349 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.B5lwUVIb0e ++++ cat /tmp/tmp.EkJO1Nc349 ++++ rm /tmp/tmp.B5lwUVIb0e /tmp/tmp.EkJO1Nc349 ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Tyu7F8pinT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gYBD0Hh7TQ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Tyu7F8pinT +++ cat /tmp/tmp.gYBD0Hh7TQ +++ rm /tmp/tmp.Tyu7F8pinT /tmp/tmp.gYBD0Hh7TQ +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-4603 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KT0Yr0GITc +++ mktemp ++ local LAST_ERR=/tmp/tmp.WJzs2WsqoI ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.KT0Yr0GITc ++ cat /tmp/tmp.WJzs2WsqoI ++ rm /tmp/tmp.KT0Yr0GITc /tmp/tmp.WJzs2WsqoI ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.Lh876Db9nl ++ mktemp + local LAST_ERR=/tmp/tmp.Uqs85BCId1 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Lh876Db9nl + cat /tmp/tmp.Uqs85BCId1 + rm /tmp/tmp.Lh876Db9nl /tmp/tmp.Uqs85BCId1 + return 0 ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mq3kF1MMby +++ mktemp ++ local LAST_ERR=/tmp/tmp.JyGw0QOg4W ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.mq3kF1MMby ++ cat /tmp/tmp.JyGw0QOg4W ++ rm /tmp/tmp.mq3kF1MMby /tmp/tmp.JyGw0QOg4W ++ return 0 + replica_pod_name=some-name-repl1-7987df9b7-jq69v ++ grep some-name-repl1-7987df9b7-jq69v /tmp/tmp.UXRFYGRGBI/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl1-7987df9b7-jq69v | 10.109.89.59 | streaming ]] ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9ab8V3gj6x +++ mktemp ++ local LAST_ERR=/tmp/tmp.hgno3XQhi3 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.9ab8V3gj6x ++ cat /tmp/tmp.hgno3XQhi3 ++ rm /tmp/tmp.9ab8V3gj6x /tmp/tmp.hgno3XQhi3 ++ return 0 + replica_pod_IP=10.109.89.59 + compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.89.59 '' ' ' + local command_id=select-2 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.89.59 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-2.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.89.59 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.89.59 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Acm7NCEpCa +++ mktemp ++ local LAST_ERR=/tmp/tmp.1hb6REMF9V ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Acm7NCEpCa ++ cat /tmp/tmp.1hb6REMF9V ++ rm /tmp/tmp.Acm7NCEpCa /tmp/tmp.1hb6REMF9V ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.89.59 ' ++ mktemp + local LAST_OUT=/tmp/tmp.AJO5zyaNWx ++ mktemp + local LAST_ERR=/tmp/tmp.wzPeekvaK3 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.89.59 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.AJO5zyaNWx + cat /tmp/tmp.wzPeekvaK3 + rm /tmp/tmp.AJO5zyaNWx /tmp/tmp.wzPeekvaK3 + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.UXRFYGRGBI/select-2.sql + for r in '$(seq 1 3)' + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' ++ mktemp + local LAST_OUT=/tmp/tmp.Pmd96dNkTD ++ mktemp + local LAST_ERR=/tmp/tmp.zDT2DjT2rl + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Pmd96dNkTD perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.zDT2DjT2rl + rm /tmp/tmp.Pmd96dNkTD /tmp/tmp.zDT2DjT2rl + return 0 + sleep 25 + wait_deployment some-name-repl2 + local name=some-name-repl2 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl2 + sleep 35 + check_replica some-name-repl2 some-name select-2 + local name=some-name-repl2 + local cluster=some-name + local sql_cmp=select-2 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7WcJAIHZDv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BaFAuxBX9z +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7WcJAIHZDv +++ cat /tmp/tmp.BaFAuxBX9z +++ rm /tmp/tmp.7WcJAIHZDv /tmp/tmp.BaFAuxBX9z +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UGkGAcTESv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VlrlZMGWXk +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.UGkGAcTESv +++ cat /tmp/tmp.VlrlZMGWXk +++ rm /tmp/tmp.UGkGAcTESv /tmp/tmp.VlrlZMGWXk +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bV9qmi4Zzw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NMgEqt4Ot0 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bV9qmi4Zzw +++ cat /tmp/tmp.NMgEqt4Ot0 +++ rm /tmp/tmp.bV9qmi4Zzw /tmp/tmp.NMgEqt4Ot0 +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.qN7FXXbav1 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.6Bg7d6aUvA ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.qN7FXXbav1 ++++ cat /tmp/tmp.6Bg7d6aUvA ++++ rm /tmp/tmp.qN7FXXbav1 /tmp/tmp.6Bg7d6aUvA ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IjA85C2XK2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0inGiv3fbs +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.IjA85C2XK2 +++ cat /tmp/tmp.0inGiv3fbs +++ rm /tmp/tmp.IjA85C2XK2 /tmp/tmp.0inGiv3fbs +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-4603 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6KnHPiJ7VR +++ mktemp ++ local LAST_ERR=/tmp/tmp.LwnNL5eSjy ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.6KnHPiJ7VR ++ cat /tmp/tmp.LwnNL5eSjy ++ rm /tmp/tmp.6KnHPiJ7VR /tmp/tmp.LwnNL5eSjy ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.BuKdESd9pV ++ mktemp + local LAST_ERR=/tmp/tmp.NVsYt1gSTI + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.BuKdESd9pV + cat /tmp/tmp.NVsYt1gSTI + rm /tmp/tmp.BuKdESd9pV /tmp/tmp.NVsYt1gSTI + return 0 ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LovmT7OUtg +++ mktemp ++ local LAST_ERR=/tmp/tmp.vpg2TrpTzr ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.LovmT7OUtg ++ cat /tmp/tmp.vpg2TrpTzr ++ rm /tmp/tmp.LovmT7OUtg /tmp/tmp.vpg2TrpTzr ++ return 0 + replica_pod_name=some-name-repl2-57cf7d6dcf-l6zr6 ++ grep some-name-repl2-57cf7d6dcf-l6zr6 /tmp/tmp.UXRFYGRGBI/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl2-57cf7d6dcf-l6zr6 | 10.109.88.64 | streaming ]] ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cwQS8vN1KG +++ mktemp ++ local LAST_ERR=/tmp/tmp.esijQtV84P ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.cwQS8vN1KG ++ cat /tmp/tmp.esijQtV84P ++ rm /tmp/tmp.cwQS8vN1KG /tmp/tmp.esijQtV84P ++ return 0 + replica_pod_IP=10.109.88.64 + compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.88.64 '' ' ' + local command_id=select-2 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.88.64 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-2.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.88.64 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.88.64 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.biL0UYfBiS +++ mktemp ++ local LAST_ERR=/tmp/tmp.vShYUTcxt1 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.biL0UYfBiS ++ cat /tmp/tmp.vShYUTcxt1 ++ rm /tmp/tmp.biL0UYfBiS /tmp/tmp.vShYUTcxt1 ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.88.64 ' ++ mktemp + local LAST_OUT=/tmp/tmp.EqACw0fxZ1 ++ mktemp + local LAST_ERR=/tmp/tmp.fwVMPbBpf5 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.88.64 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.EqACw0fxZ1 + cat /tmp/tmp.fwVMPbBpf5 + rm /tmp/tmp.EqACw0fxZ1 /tmp/tmp.fwVMPbBpf5 + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.UXRFYGRGBI/select-2.sql + for r in '$(seq 1 3)' + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]' ++ mktemp + local LAST_OUT=/tmp/tmp.vPYiJ4xNOr ++ mktemp + local LAST_ERR=/tmp/tmp.ruSAlqmMu1 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.vPYiJ4xNOr perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.ruSAlqmMu1 + rm /tmp/tmp.vPYiJ4xNOr /tmp/tmp.ruSAlqmMu1 + return 0 + sleep 25 + wait_deployment some-name-repl3 + local name=some-name-repl3 + local target_namespace=recreate-4603 + sleep 10 + set +o xtrace some-name-repl3 + sleep 35 + check_replica some-name-repl3 some-name select-2 + local name=some-name-repl3 + local cluster=some-name + local sql_cmp=select-2 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-4603 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tyQDyVJUKu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zZybGLP7TU +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.tyQDyVJUKu +++ cat /tmp/tmp.zZybGLP7TU +++ rm /tmp/tmp.tyQDyVJUKu /tmp/tmp.zZybGLP7TU +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.huQVXhC9xO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yfkb09eWxz +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.huQVXhC9xO +++ cat /tmp/tmp.yfkb09eWxz +++ rm /tmp/tmp.huQVXhC9xO /tmp/tmp.yfkb09eWxz +++ return 0 +++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TLY3Y9i7AO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8O3dUPHSMZ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TLY3Y9i7AO +++ cat /tmp/tmp.8O3dUPHSMZ +++ rm /tmp/tmp.TLY3Y9i7AO /tmp/tmp.8O3dUPHSMZ +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.OcQhMJfen3 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.4MnUVQVYyP ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++++ kubectl -n recreate-4603 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.OcQhMJfen3 ++++ cat /tmp/tmp.4MnUVQVYyP ++++ rm /tmp/tmp.OcQhMJfen3 /tmp/tmp.4MnUVQVYyP ++++ return 0 +++ kubectl_bin -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZpLjkEz0fK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OXw71NBT6W +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox +++ kubectl -n recreate-4603 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ZpLjkEz0fK +++ cat /tmp/tmp.OXw71NBT6W +++ rm /tmp/tmp.ZpLjkEz0fK /tmp/tmp.OXw71NBT6W +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-4603 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-4603 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QGhK4twWOU +++ mktemp ++ local LAST_ERR=/tmp/tmp.HMsK2SJ055 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.QGhK4twWOU ++ cat /tmp/tmp.HMsK2SJ055 ++ rm /tmp/tmp.QGhK4twWOU /tmp/tmp.HMsK2SJ055 ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.PKTEeh3N7J ++ mktemp + local LAST_ERR=/tmp/tmp.s2yrHTntz9 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-4603.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.PKTEeh3N7J + cat /tmp/tmp.s2yrHTntz9 + rm /tmp/tmp.PKTEeh3N7J /tmp/tmp.s2yrHTntz9 + return 0 ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wquSKg4hQN +++ mktemp ++ local LAST_ERR=/tmp/tmp.JGPlCpdP94 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.wquSKg4hQN ++ cat /tmp/tmp.JGPlCpdP94 ++ rm /tmp/tmp.wquSKg4hQN /tmp/tmp.JGPlCpdP94 ++ return 0 + replica_pod_name=some-name-repl3-97b8b7b55-slhdx ++ grep some-name-repl3-97b8b7b55-slhdx /tmp/tmp.UXRFYGRGBI/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl3-97b8b7b55-slhdx | 10.109.90.38 | streaming ]] ++ kubectl_bin -n recreate-4603 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.E96E3poJh0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ybP7q9DTi2 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.E96E3poJh0 ++ cat /tmp/tmp.ybP7q9DTi2 ++ rm /tmp/tmp.E96E3poJh0 /tmp/tmp.ybP7q9DTi2 ++ return 0 + replica_pod_IP=10.109.90.38 + compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.90.38 '' ' ' + local command_id=select-2 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.90.38 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-2.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.109.90.38 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.109.90.38 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7IWz6Y0gz0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8WuNOTBPYh ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl -n recreate-4603 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.7IWz6Y0gz0 ++ cat /tmp/tmp.8WuNOTBPYh ++ rm /tmp/tmp.7IWz6Y0gz0 /tmp/tmp.8WuNOTBPYh ++ return 0 + local client_container=pg-client-76df9df876-r4xl7 + kubectl_bin exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.90.38 ' ++ mktemp + local LAST_OUT=/tmp/tmp.hzZJLTLIH6 ++ mktemp + local LAST_ERR=/tmp/tmp.zKtMXGGHnI + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl exec pg-client-76df9df876-r4xl7 -n recreate-4603 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.109.90.38 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.hzZJLTLIH6 + cat /tmp/tmp.zKtMXGGHnI + rm /tmp/tmp.hzZJLTLIH6 /tmp/tmp.zKtMXGGHnI + return 0 + '[' '!' -s /tmp/tmp.UXRFYGRGBI/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.UXRFYGRGBI/select-2.sql ++ kubectl_bin get pods --selector=role=master,service-name=some-name,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MU3KeYfFka +++ mktemp ++ local LAST_ERR=/tmp/tmp.BvBpk0MSe5 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl get pods --selector=role=master,service-name=some-name,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.MU3KeYfFka ++ cat /tmp/tmp.BvBpk0MSe5 ++ rm /tmp/tmp.MU3KeYfFka /tmp/tmp.BvBpk0MSe5 ++ return 0 + master_container=some-name-696f799f57-lgjsx + kubectl_bin get configmap/some-name-pgha-config -o yaml + yq w - postgresql.parameters.jit false + yq r - 'data[some-name-dcs-config]' ++ mktemp + local LAST_OUT=/tmp/tmp.w6Gzqsa3oh ++ mktemp + local LAST_ERR=/tmp/tmp.uFNK9AetP0 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get configmap/some-name-pgha-config -o yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.w6Gzqsa3oh + cat /tmp/tmp.uFNK9AetP0 + rm /tmp/tmp.w6Gzqsa3oh /tmp/tmp.uFNK9AetP0 + return 0 + kubectl_bin apply -f - + kubectl_bin get configmap/some-name-pgha-config -o yaml ++ mktemp ++ mktemp ++ cat /tmp/tmp.UXRFYGRGBI/some-name-pgha-config.patch + local LAST_OUT=/tmp/tmp.hOQsN0Nfeu + local LAST_OUT=/tmp/tmp.v0MXXAZkfd ++ mktemp + yq w - 'data[some-name-dcs-config]' 'postgresql: parameters: jit: false unix_socket_directories: /tmp wal_level: logical archive_mode: true archive_command: source /opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-set-env.sh && pgbackrest archive-push "%p" archive_timeout: 60 log_directory: pg_log shared_buffers: 128MB temp_buffers: 8MB log_min_duration_statement: 60000 log_statement: none work_mem: 4MB max_wal_senders: 6 shared_preload_libraries: pg_stat_monitor,pgaudit ssl: "on" ssl_cert_file: /pgconf/tls/tls.crt ssl_key_file: /pgconf/tls/tls.key ssl_ca_file: /pgconf/tls/ca.crt use_slots: false recovery_conf: restore_command: source /opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-set-env.sh && pgbackrest archive-get %f "%p" use_pg_rewind: true' + local LAST_ERR=/tmp/tmp.AGGdM2OoZt + local exit_status=0 + local timeout=4 + set +o errexit ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.rjVV16SVii + local exit_status=0 + local timeout=4 + set +o errexit + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl apply -f - ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get configmap/some-name-pgha-config -o yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.hOQsN0Nfeu + cat /tmp/tmp.rjVV16SVii + rm /tmp/tmp.hOQsN0Nfeu /tmp/tmp.rjVV16SVii + return 0 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.v0MXXAZkfd configmap/some-name-pgha-config configured + cat /tmp/tmp.AGGdM2OoZt Warning: resource configmaps/some-name-pgha-config is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.v0MXXAZkfd /tmp/tmp.AGGdM2OoZt + return 0 ++ kubectl_bin get configmap/some-name-pgha-config -o yaml ++ yq r - 'data[some-name-dcs-config]' ++ yq r - postgresql.parameters.jit +++ mktemp ++ local LAST_OUT=/tmp/tmp.GgvSpuElGR +++ mktemp ++ local LAST_ERR=/tmp/tmp.coO8zrx3uw ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl get configmap/some-name-pgha-config -o yaml ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.GgvSpuElGR ++ cat /tmp/tmp.coO8zrx3uw ++ rm /tmp/tmp.GgvSpuElGR /tmp/tmp.coO8zrx3uw ++ return 0 + CONF_MAP_JIT=false ++ kubectl_bin exec some-name-696f799f57-lgjsx -- bash -c 'patronictl show-config | grep jit | grep -Eo "true|false" ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xF4rJk7ZG8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.waA7r9vXSb ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl exec some-name-696f799f57-lgjsx -- bash -c 'patronictl show-config | grep jit | grep -Eo "true|false" ' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.xF4rJk7ZG8 ++ cat /tmp/tmp.waA7r9vXSb ++ rm /tmp/tmp.xF4rJk7ZG8 /tmp/tmp.waA7r9vXSb ++ return 0 + PATRONI_JIT=false + [[ false != false ]] + destroy recreate-4603 + grep -v level=info + grep -v level=debug + grep -v 'Getting pods from source' + grep -v 'Getting tasks for pod' + grep -v 'the object has been modified' + grep -v 'get backup status: Job.batch' + sort -u + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + tee /tmp/tmp.UXRFYGRGBI/operator.log ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fkKBv61ILY +++ mktemp ++ local LAST_ERR=/tmp/tmp.ocaLTBLaEs ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.fkKBv61ILY ++ cat /tmp/tmp.ocaLTBLaEs ++ rm /tmp/tmp.fkKBv61ILY /tmp/tmp.ocaLTBLaEs ++ return 0 + kubectl_bin logs postgres-operator-58d9cddfdb-82fnk -c operator ++ mktemp + local LAST_OUT=/tmp/tmp.zbDRFTdDtM ++ mktemp + local LAST_ERR=/tmp/tmp.Ju5KXttjZB + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl logs postgres-operator-58d9cddfdb-82fnk -c operator + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.zbDRFTdDtM + cat /tmp/tmp.Ju5KXttjZB + rm /tmp/tmp.zbDRFTdDtM /tmp/tmp.Ju5KXttjZB + return 0 ] { } }, ] } }, { } }, ] { } }, { } ] } { } }, ] } ] { } }, }, { ] { } }, ] ] ], { } }, }, { }, { },{ }] }], ] { } }, , ], { } }, }], { } }, }], } ], } }, } }, } }, { } }, }, 2024/04/24 18:39:08 INF 1 (localhost:4150) connecting to nsqd 2024/04/24 18:39:57 INF 2 (localhost:4150) connecting to nsqd 2024/04/24 18:39:58 INF 3 (localhost:4150) connecting to nsqd 2024/04/24 18:40:04 INF 4 (localhost:4150) connecting to nsqd 2024/04/24 18:40:32 INF 5 (localhost:4150) connecting to nsqd 2024/04/24 18:40:32 INF 6 (localhost:4150) connecting to nsqd 2024/04/24 18:40:33 INF 7 (localhost:4150) connecting to nsqd 2024/04/24 18:41:34 INF 8 (localhost:4150) connecting to nsqd 2024/04/24 18:41:52 INF 9 (localhost:4150) connecting to nsqd 2024/04/24 18:43:02 INF 10 (localhost:4150) connecting to nsqd 2024/04/24 18:44:21 INF 11 (localhost:4150) connecting to nsqd 2024/04/24 18:45:41 INF 12 (localhost:4150) connecting to nsqd 2024/04/24 18:52:01 INF 13 (localhost:4150) connecting to nsqd 2024/04/24 18:52:50 INF 14 (localhost:4150) connecting to nsqd 2024/04/24 18:53:40 INF 15 (localhost:4150) connecting to nsqd 2024/04/24 18:54:25 INF 16 (localhost:4150) connecting to nsqd 2024/04/24 18:54:42 INF 17 (localhost:4150) connecting to nsqd 2024/04/24 18:54:47 INF 18 (localhost:4150) connecting to nsqd 2024/04/24 18:55:42 INF 19 (localhost:4150) connecting to nsqd 2024/04/24 18:56:23 INF 20 (localhost:4150) connecting to nsqd 2024/04/24 18:56:24 INF 21 (localhost:4150) connecting to nsqd 2024/04/24 18:56:32 INF 22 (localhost:4150) connecting to nsqd 2024/04/24 18:56:46 INF 23 (localhost:4150) connecting to nsqd 2024/04/24 18:56:46 INF 24 (localhost:4150) connecting to nsqd 2024/04/24 18:56:46 INF 25 (localhost:4150) connecting to nsqd 2024/04/24 18:57:25 INF 26 (localhost:4150) connecting to nsqd 2024/04/24 18:58:43 INF 27 (localhost:4150) connecting to nsqd 2024/04/24 19:00:03 INF 28 (localhost:4150) connecting to nsqd "accessModes": [ "affinity": "affinity": { "affinity": { "podAntiAffinity": { "allowPrivilegeEscalation": false, "annotations": {"keep-backups":"false","keep-data":"false"}, "apiVersion": "apps/v1", "apiVersion": "batch/v1", "apiVersion": "v1", "apiVersion": "v1", "apiVersion": "v1", "backrest-command": "backup", "backrest-command": "stanza-create", "claimName": "some-name-pgbr-repo" "command": [ "configMap": { "configMap": { "configMapKeyRef": { "containerName": "database", "containerPort": 2022, "containerPort": 5432, "containerPort": 8009, "containers": [ "containers": [{ "cpu": "1", "cpu": "200m", "cpu": "500m", "crunchydata" "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "defaultMode": 288 "defaultMode": 288, "defaultMode": 288, "defaultMode": 420, "deployment-name": "some-name" "deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0","name": "some-name","pg-cluster": "some-name" "deployment-name": "some-name-repl1" "deployment-name": "some-name-repl1","pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name" "deployment-name": "some-name-repl2" "deployment-name": "some-name-repl2","pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name" "deployment-name": "some-name-repl3" "divisor": "1m", "dnsPolicy": "ClusterFirst" "downwardAPI": { "emptyDir": { "emptyDir": { "emptyDir": { "emptyDir": { "env": [ "env": [{ "exec": { "fieldPath": "metadata.annotations" "fieldPath": "metadata.labels" "fieldPath": "metadata.labels['crunchy-pgha-scope']" "fieldPath": "metadata.namespace" "fieldRef": { "fieldRef": { "fsGroup": 2, I0424 18:38:46.096733 1 shared_informer.go:240] Waiting for caches to sync for namespace I0424 18:38:46.197170 1 shared_informer.go:247] Caches are synced for namespace I0424 18:38:46.321661 1 shared_informer.go:240] Waiting for caches to sync for recreate-4603 I0424 18:38:46.422609 1 shared_informer.go:247] Caches are synced for recreate-4603 "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer", "image": "perconalab/percona-postgresql-operator:main-ppg14-postgres-ha", "image": "perconalab/percona-postgresql-operator:PR-723-7ed19e708-pgo-rmdata", "imagePullPolicy": "IfNotPresent" "imagePullPolicy": "IfNotPresent", "initialDelaySeconds": 15 "initialDelaySeconds": 30, "items": [ "items": [ "items": [ "key": "config", "key": "config", "key": "crunchy-pgbouncer", "key": "init" "key": "password" "key": "pg-cluster", "key": "pgo-backrest", "key": "pgo-backrest-job", "key": "pgo-pg-database", "key": "pg-pod-anti-affinity", "key": "tls.crt", "key": "tls.key", "key": "vendor", "kind": "Deployment", "kind": "Job", "kind": "PersistentVolumeClaim", "kind": "Service", "labels": { "labels": { "labels": { "labelSelector": { "livenessProbe": { "matchExpressions": [ "matchLabels": { "matchLabels": { "maxSurge": 1 "maxUnavailable": 1, "medium": "Memory", "medium": "Memory" "medium": "Memory", "medium": "Memory", "medium": "Memory", "memory": "128Mi" "memory": "256Mi" "memory": "48Mi" "metadata": { "metadata": { "metadata": { "mountPath": "/backrestrepo", "mountPath": "/dev/shm", "mountPath": "/etc/pgbackrest" "mountPath": "/etc/pgbackrest/conf.d", "mountPath": "/etc/podinfo", "mountPath": "/etc/ssh", "mountPath": "/etc/ssh", "mountPath": "/home/pgbackrest" "mountPath": "/pgconf", "mountPath": "/pgconf/", "mountPath": "/pgconf/pgreplicator", "mountPath": "/pgconf/pgsuper", "mountPath": "/pgconf/pguser", "mountPath": "/pgconf/tls", "mountPath": "/pgconf/tls/pgbouncer", "mountPath": "/pgconf/tls-replication", "mountPath": "/pgdata", "mountPath": "/sshd", "mountPath": "/sshd", "mountPath": "/tmp", "mountPath": "/tmp", "mountPath": "/tmp" "name": "backrest", "name": "backrest-backup-some-name", "name": "backrestrepo", "name": "backrestrepo", "name": "BACKREST_SKIP_CREATE_STANZA", "name": "cluster1-replication-ssl-keypair", "name": "cluster1-ssl-ca" "name": "cluster1-ssl-ca" "name": "cluster1-ssl-keypair" "name": "cluster1-ssl-keypair" "name": "COMMAND", "name": "COMMAND_OPTS", "name": "database", "name": "dshm" "name": "dshm", "name": "ENABLE_SSHD", "name": "IS_BACKUP", "name": "IS_REPLICA", "name": "LD_PRELOAD", "name": "LD_PRELOAD", "name": "MODE", "name": "NAMESPACE", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_PASSWD", "name": "NSS_WRAPPER_PASSWD", "name": "patroni" "name": "PATRONI_KUBERNETES_LABELS", "name": "PATRONI_KUBERNETES_NAMESPACE", "name": "PATRONI_KUBERNETES_SCOPE_LABEL", "name": "PATRONI_LOG_LEVEL", "name": "PATRONI_POSTGRESQL_DATA_DIR", "name": "PATRONI_SCOPE", "name": "pgbackrest" "name": "pgbackrest-conf", "name": "pgbackrest-conf", "name": "pgbackrest-config" "name": "pgbackrest-config", "name": "PGBACKREST_DB_HOST", "name": "PGBACKREST_DB_PATH", "name": "PGBACKREST_DB_PATH", "name": "pgbackrest-home", "name": "pgbackrest-home", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_REPO1_HOST", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_STANZA", "name": "PGBACKREST_STANZA", "name": "pgbouncer" "name": "pgbouncer", "name": "pgbouncer-conf", "name": "pgbouncer-conf", "name": "PG_CLUSTER", "name": "pgconf-volume" "name": "pgconf-volume", "name": "pgdata", "name": "pgdata", "name": "PGHA_DATABASE", "name": "PGHA_INIT", "name": "PGHA_PASSWORD_TYPE", "name": "PGHA_PGBACKREST", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_S3_VERIFY_TLS", "name": "PGHA_PG_PORT", "name": "PGHA_REPLICA_REINIT_ON_START_FAIL", "name": "PGHA_SCOPE", "name": "PGHA_STANDBY", "name": "PGHA_SYNC_REPLICATION", "name": "PGHA_TLS_ENABLED", "name": "PGHA_TLS_ONLY", "name": "PGHA_USER", "name": "PGHOST", "name": "PG_PASSWORD", "name": "PG_PRIMARY_SERVICE_NAME", "name": "PITR_TARGET", "name": "podinfo" "name": "podinfo", "name": "PODNAME", "name": "postgres" "name": "primary-volume" "name": "primary-volume", "name": "REMOVE_BACKUP", "name": "REMOVE_DATA", "name": "REPLICA_NAME", "name": "report", "name": "rmdata", "name": "root-volume" "name": "root-volume", "name": "some-name", "name": "some-name", "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo" "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer-cm" "name": "some-name-pgbouncer-secret", "name": "some-name-pgbouncer-secret", "name": "some-name-pgbr-repo", "name": "some-name","pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0" "name": "some-name-pgha-config", "name": "some-name-pgha-config", "name": "some-name-repl1", "name": "some-name-repl1", "name": "some-name-repl2", "name": "some-name-repl2", "name": "some-name-repl3", "name": "some-name-repl3", "name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2","pgo-version": "1.6.0" "name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl3","pgo-version": "1.6.0" "name": "some-name-rmdata-cklp", "name": "some-name-rmdata-cklp", "name": "some-name-stanza-create", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "sshd", "name": "sshd", "name": "sshd", "name": "sshd", "name": "tls-pgbouncer" "name": "tls-pgbouncer", "name": "tls-replication" "name": "tls-replication", "name": "tls-server" "name": "tls-server", "name": "tmp" "name": "tmp" "name": "tmp", "name": "tmp", "name": "tmp", "name": "user-volume" "name": "user-volume", "nodePort": 0 "operator": "Exists" "operator": "In", "/opt/crunchy/bin/postgres-ha/health/pgha-liveness.sh" "/opt/crunchy/bin/postgres-ha/health/pgha-readiness.sh" "optional": true "path": "annotations" "path": "cpu_limit", "path": "cpu_request", "path": "labels" "path": "mem_limit", "path": "mem_request", "path": "ssh_config" "path": "ssh_config" "path": "tls-replication.crt" "path": "tls-replication.key" "periodSeconds": 15, "persistentVolumeClaim": { "persistentVolumeClaim":{"claimName":"some-name"} "persistentVolumeClaim":{"claimName":"some-name-repl1"} "persistentVolumeClaim":{"claimName":"some-name-repl2"} "persistentVolumeClaim":{"claimName":"some-name-repl3"} "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name","deployment-name": "some-name-repl1","pgo-version": "1.6.0","name": "some-name-replica" "pgo-backrest-job": "true", "pgo-backrest-repo": "true" "pgo-backrest-repo": "true", "pgo-backrest-repo": "true" "pgo-backrest": "true", "pgo-pg-database": "true", "pgo-pg-database": "true", "pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0","name": "some-name","pg-cluster": "some-name","deployment-name": "some-name" "pgo-version": "1.6.0","name": "some-name","pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name" "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1" "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1" "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2" "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2" "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl3" "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl3" "pg-pod-anti-affinity": "preferred", "pg-pod-anti-affinity": "preferred", "pgrmdata": "true", "pgrmdata": "true", "podAffinityTerm": { "podAntiAffinity": { "podAntiAffinity": { "podAntiAffinity": { "port": 2022, "ports": [{ "ports": [{ "preferredDuringSchedulingIgnoredDuringExecution": [ "privileged": false, "projected": { "projected": { "projected": { "projected": { "sources": [] } "protocol": "TCP", "protocol": "TCP", "readinessProbe": { "readOnly": false "readOnly": false "readOnly": false "readOnlyRootFilesystem": true "readOnly": true "readOnly": true "readOnly": true "ReadWriteOnce" "replicas": 0, "replicas": 1, "replicas": 3, "requests": { "requests": { "resourceFieldRef": { "resource": "limits.cpu" "resource": "limits.memory" "resource": "requests.cpu" "resource": "requests.memory" "resources": { "resources": { "restartPolicy": "Always", "restartPolicy": "Never" "rollingUpdate": { "runAsNonRoot": true "secret": { "secret": { "secret": { "secret": { "secret": { "secretKeyRef": { "secretName": "some-name-backrest-repo-config" "secretName": "some-name-backrest-repo-config", "secretName": "some-name-backrest-repo-config" "secretName": "some-name-backrest-repo-config", "secretName": "some-name-postgres-secret" "secretName": "some-name-primaryuser-secret" "secretName": "some-name-some-name-secret" "securityContext": { "securityContext": { "securityContext": {}, "securityContext": {"fsGroup": 26,"supplementalGroups": [1001]}, "securityContext": {"runAsNonRoot":true,"fsGroup":26}, "selector": { "selector": { "serviceAccountName": "pgo-backrest", "serviceAccountName": "pgo-default", "serviceAccountName": "pgo-pg", "serviceAccountName": "pgo-target", "service-name": "some-name-backrest-shared-repo", "service-name": "some-name-backrest-shared-repo", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer", "sessionAffinity": "None" "sizeLimit": "128Ki" "sizeLimit": "16Mi" "sizeLimit": "1Mi" "sizeLimit": "2Mi" "sizeLimit": "64Mi" "sizeLimit": "64Mi" "some-name" "sources": [ "sources": [ "sources": [ "spec": { "spec": { "spec": { "storage": "1Gi" "strategy": { "targetPort": 2022, "template": { time="2024-04-24T18:38:45Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.6.0 time="2024-04-24T18:39:12Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-24T18:39:17Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-24T18:39:57Z" level=error msg="Error completing workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-24T18:39:57Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-24T18:40:33Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-24T18:40:33Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-24T18:40:46Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0 time="2024-04-24T18:40:46Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:40:46Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0 time="2024-04-24T18:44:21Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-24T18:45:41Z" level=warning msg="crv1 pgreplica some-name-repl3 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-24T18:45:46Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0 time="2024-04-24T18:45:46Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:45:46Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0 time="2024-04-24T18:48:22Z" level=warning msg="readiness timeout reached for deployment \"some-name-repl1\"" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.RollingUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:116" version=1.6.0 time="2024-04-24T18:48:26Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:48:30Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:48:34Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:48:38Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:48:42Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:48:46Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:48:50Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:48:55Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:48:58Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:49:02Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:49:06Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:49:10Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:49:14Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:49:18Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:49:22Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:49:22Z" level=warning msg="readiness timeout reached for start up of cluster \"some-name\" instance \"some-name-repl1\"" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.RollingUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:122" version=1.6.0 time="2024-04-24T18:49:46Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.6.0 time="2024-04-24T18:49:46Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:49:46Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.6.0 time="2024-04-24T18:51:02Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:51:02Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.6.0 time="2024-04-24T18:54:25Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:26Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:27Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:28Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:29Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:31Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:35Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:36Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:40Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:41Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:42Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0 time="2024-04-24T18:54:47Z" level=error msg="jobs.batch \"some-name-rmdata-cklp\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0 time="2024-04-24T18:55:46Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-24T18:55:51Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0 time="2024-04-24T18:56:23Z" level=error msg="Error completing workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0 time="2024-04-24T18:56:23Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0 time="2024-04-24T18:56:46Z" level=error msg="no pod found for \"some-name\"" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0 time="2024-04-24T18:56:46Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-24T18:56:47Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0 time="2024-04-24T18:56:47Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:56:47Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0 time="2024-04-24T18:56:47Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-24T18:58:43Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 time="2024-04-24T18:58:46Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0 time="2024-04-24T18:58:46Z" level=error msg="pod some-name-repl2-57cf7d6dcf-l6zr6 does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0 time="2024-04-24T18:58:46Z" level=error msg="pod some-name-repl2-57cf7d6dcf-l6zr6 does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0 time="2024-04-24T19:00:03Z" level=warning msg="crv1 pgreplica some-name-repl3 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0 "timeoutSeconds": 10 "topologyKey": "kubernetes.io/hostname" "type": "ClusterIP", "type": "RollingUpdate", "value": "" "value": "5432" "value": "5432" "value": "/backrestrepo/some-name-backrest-shared-repo" "value": "/backrestrepo/some-name-backrest-shared-repo" "value": "backup" "value": "crunchy-pgha-scope" "value": "db" "value": "db" "value": " --db-host=10.109.90.30 --db-path=/pgdata/some-name" "value": " --db-host=10.109.90.36 --db-path=/pgdata/some-name" "value": "false" "value": "false" "valueFrom": { "value": "INFO" "value": "pgbackrest" "value": "pgbackrest-repo" "value": "/pgdata/some-name" "value": "/pgdata/some-name" "value": "/pgdata/some-name-repl1" "value": "/pgdata/some-name-repl1" "value": "/pgdata/some-name-repl2" "value": "/pgdata/some-name-repl2" "value": "/pgdata/some-name-repl3" "value": "/pgdata/some-name-repl3" "value": "posix" "value": "posix" "value": "postgres" "values": [ "value": "some-name" "value": "some-name-backrest-shared-repo" "value": "some-name-backrest-shared-repo-6f8f947dbc-fgdj2" "value": "some-name-backrest-shared-repo-6f8f947dbc-qw8pg" "value": "stanza-create" "value": "/tmp" "value": "/tmp" "value": "/tmp/nss_wrapper/pgbackrest-repo/group" "value": "/tmp/nss_wrapper/pgbackrest-repo/passwd" "value": "/tmp/nss_wrapper/postgres/group" "value": "/tmp/nss_wrapper/postgres/passwd" "value": "true" "value": "true" "value": "--type=full --db-host=10.109.90.30 --db-path=/pgdata/some-name" "value": "--type=full --db-host=10.109.90.36 --db-path=/pgdata/some-name" "value": "/usr/lib64/libnss_wrapper.so" "value": "/usr/lib64/libnss_wrapper.so" "value": "{vendor: \"crunchydata\"}" "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "volumeMounts": [ "volumeMounts": [{ "volumes": [ "volumes": [{ "weight": 1, + destroy_operator + kubectl_bin delete -f /tmp/tmp.UXRFYGRGBI/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.qDC9oSvbus ++ mktemp + local LAST_ERR=/tmp/tmp.5Iq12D8fGQ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete -f /tmp/tmp.UXRFYGRGBI/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete -f /tmp/tmp.UXRFYGRGBI/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete -f /tmp/tmp.UXRFYGRGBI/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.qDC9oSvbus + cat /tmp/tmp.5Iq12D8fGQ Error from server (NotFound): error when deleting "/tmp/tmp.UXRFYGRGBI/operator.yaml": serviceaccounts "pgo-deployer-sa" not found Error from server (NotFound): error when deleting "/tmp/tmp.UXRFYGRGBI/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): error when deleting "/tmp/tmp.UXRFYGRGBI/operator.yaml": configmaps "pgo-deployer-cm" not found Error from server (NotFound): error when deleting "/tmp/tmp.UXRFYGRGBI/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found Error from server (NotFound): error when deleting "/tmp/tmp.UXRFYGRGBI/operator.yaml": jobs.batch "pgo-deploy" not found + rm /tmp/tmp.qDC9oSvbus /tmp/tmp.5Iq12D8fGQ + return 1 + true + yq w -d4 /tmp/tmp.UXRFYGRGBI/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall + kubectl_bin apply -f /tmp/tmp.UXRFYGRGBI/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.dvSVeyVT87 ++ mktemp + local LAST_ERR=/tmp/tmp.4I4ZRSAdUd + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl apply -f /tmp/tmp.UXRFYGRGBI/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.dvSVeyVT87 serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.4I4ZRSAdUd + rm /tmp/tmp.dvSVeyVT87 /tmp/tmp.4I4ZRSAdUd + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=recreate-4603 + retry=0 + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-4603 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.UXRFYGRGBI/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.siiLYwODHI ++ mktemp + local LAST_ERR=/tmp/tmp.22GrocChsr + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete -f /tmp/tmp.UXRFYGRGBI/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.siiLYwODHI serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.22GrocChsr + rm /tmp/tmp.siiLYwODHI /tmp/tmp.22GrocChsr + return 0 + kubectl_bin get ns + awk '{print$1}' + xargs kubectl delete ns --grace-period=0 --force=true + egrep -v '^kube-|^default|Terminating|openshift|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.2DRSwfNCiF ++ mktemp + local LAST_ERR=/tmp/tmp.ZqWu930hqR + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2DRSwfNCiF + cat /tmp/tmp.ZqWu930hqR + rm /tmp/tmp.2DRSwfNCiF /tmp/tmp.ZqWu930hqR + return 0 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. namespace "recreate-4603" force deleted + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.bWov4fWwgl ++ mktemp + local LAST_ERR=/tmp/tmp.q6bFKsYyBd + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.bWov4fWwgl + cat /tmp/tmp.q6bFKsYyBd Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.bWov4fWwgl /tmp/tmp.q6bFKsYyBd + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.UQmOnNGjAH ++ mktemp + local LAST_ERR=/tmp/tmp.p6ELzKsTmy + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.UQmOnNGjAH + cat /tmp/tmp.p6ELzKsTmy Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.UQmOnNGjAH /tmp/tmp.p6ELzKsTmy + return 1 + true + rm -rf /tmp/tmp.UXRFYGRGBI