=== RUN kuttl harness.go:460: starting setup harness.go:258: running tests using configured kubeconfig. harness.go:281: Successful connection to cluster at: https://34.9.94.85 harness.go:366: running tests harness.go:77: going to run test suite with timeout of 600 seconds for each step harness.go:378: testsuite: e2e-tests/tests has 36 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-15-to-16 === PAUSE kuttl/harness/major-upgrade-15-to-16 === CONT kuttl/harness/major-upgrade-15-to-16 logger.go:42: 19:21:05 | major-upgrade-15-to-16 | Creating namespace "kuttl-test-equipped-lab" logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | + source ../../functions logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ realpath ../../.. logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ pwd logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-15-to-16 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++ test_name=major-upgrade-15-to-16 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ command -v oc logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ PG_VER=18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ printenv logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ which gdate logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ which date logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ which gsed logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++++ which sed logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | ++ oc get projects logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | + init_temp_dir logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | + deploy_operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | + local cw_prefix= logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | + destroy_operator logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 19:21:05 | major-upgrade-15-to-16/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 19:21:06 | major-upgrade-15-to-16/0-deploy-operator | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 19:21:06 | major-upgrade-15-to-16/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 19:21:06 | major-upgrade-15-to-16/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 logger.go:42: 19:21:06 | major-upgrade-15-to-16/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 19:21:06 | major-upgrade-15-to-16/0-deploy-operator | namespace "pg-operator" force deleted logger.go:42: 19:21:17 | major-upgrade-15-to-16/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 19:21:17 | major-upgrade-15-to-16/0-deploy-operator | + create_namespace pg-operator logger.go:42: 19:21:17 | major-upgrade-15-to-16/0-deploy-operator | + local namespace=pg-operator logger.go:42: 19:21:17 | major-upgrade-15-to-16/0-deploy-operator | + [[ -n '' ]] logger.go:42: 19:21:17 | major-upgrade-15-to-16/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 19:21:18 | major-upgrade-15-to-16/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 19:21:18 | major-upgrade-15-to-16/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 19:21:18 | major-upgrade-15-to-16/0-deploy-operator | namespace/pg-operator created logger.go:42: 19:21:18 | major-upgrade-15-to-16/0-deploy-operator | + cw_prefix=cw- logger.go:42: 19:21:18 | major-upgrade-15-to-16/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/crd.yaml logger.go:42: 19:21:20 | major-upgrade-15-to-16/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:21:20 | major-upgrade-15-to-16/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 19:21:22 | major-upgrade-15-to-16/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 19:21:22 | major-upgrade-15-to-16/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 19:21:22 | major-upgrade-15-to-16/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 19:21:22 | major-upgrade-15-to-16/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:21:22 | major-upgrade-15-to-16/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:21:24 | major-upgrade-15-to-16/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:21:24 | major-upgrade-15-to-16/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cw-rbac.yaml logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | + local disable_telemetry=true logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | + '[' major-upgrade-15-to-16 == telemetry-transfer ']' logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cw-operator.yaml logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 19:21:25 | major-upgrade-15-to-16/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 19:21:26 | major-upgrade-15-to-16/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 19:21:26 | major-upgrade-15-to-16/0-deploy-operator | + deploy_client logger.go:42: 19:21:26 | major-upgrade-15-to-16/0-deploy-operator | + kubectl -n kuttl-test-equipped-lab apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf/client.yaml logger.go:42: 19:21:27 | major-upgrade-15-to-16/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 19:21:27 | major-upgrade-15-to-16/0-deploy-operator | + deploy_s3_secrets logger.go:42: 19:21:27 | major-upgrade-15-to-16/0-deploy-operator | + set +o xtrace logger.go:42: 19:21:28 | major-upgrade-15-to-16/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 19:21:29 | major-upgrade-15-to-16/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 19:21:29 | major-upgrade-15-to-16/0-deploy-operator | secret/azure-secret created logger.go:42: 19:21:30 | major-upgrade-15-to-16/0-deploy-operator | secret/minio-secret created logger.go:42: 19:21:31 | major-upgrade-15-to-16/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 19:21:31 | major-upgrade-15-to-16/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 19:21:31 | major-upgrade-15-to-16/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 19:21:31 | major-upgrade-15-to-16/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 19:21:31 | major-upgrade-15-to-16/0-deploy-operator | ASSERT PASS logger.go:42: 19:21:31 | major-upgrade-15-to-16/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | starting test step 1-create-cluster logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions get_cr \ | yq eval ' .spec.postgresVersion = 15 | .spec.image = "perconalab/percona-postgresql-operator:main-ppg15-postgres" | .spec.instances[0].dataVolumeClaimSpec.resources.requests.storage = "3Gi" | .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-pgbouncer15" | .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-pgbackrest15" | .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = "pg_cron" | .spec.extensions.custom += [{"name": "pg_cron", "version": "1.6.6"}]' \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | + source ../../functions logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ realpath ../../.. logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ pwd logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-15-to-16 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++ test_name=major-upgrade-15-to-16 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ command -v oc logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export PG_VER=18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ PG_VER=18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ printenv logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:31 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ which gdate logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ which date logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ which gsed logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++++ which sed logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ++ oc get projects logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + get_cr logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + local cr_name= logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + '[' -z ']' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + cr_name=major-upgrade-15-to-16 logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + local repo_path= logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + local source_path= logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + yq eval ' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.postgresVersion = 15 | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.image = "perconalab/percona-postgresql-operator:main-ppg15-postgres" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.instances[0].dataVolumeClaimSpec.resources.requests.storage = "3Gi" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-pgbouncer15" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-pgbackrest15" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = "pg_cron" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.extensions.custom += [{"name": "pg_cron", "version": "1.6.6"}]' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + yq eval ' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .metadata.name = "major-upgrade-15-to-16" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .metadata.labels = {"e2e":"major-upgrade-15-to-16"} | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.postgresVersion = 18 | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.users += [{"name":"major-upgrade-15-to-16","password":{"type":"AlphaNumeric"}}] | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.pmm.image = "docker.io/perconalab/pmm-client:dev-latest" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.pmm.secret = "major-upgrade-15-to-16-pmm-secret" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.pmm.customClusterName = "major-upgrade-15-to-16-pmm-custom-name" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.pmm.postgresParams = "--environment=dev-postgres" logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ' /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cr.yaml logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + kubectl -n kuttl-test-equipped-lab apply -f - logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + [[ -n '' ]] logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + case $test_name in logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + yq eval -i ' logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.extensions.image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.extensions.imagePullPolicy = "Always" | logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | .spec.extensions.storage = {"type": "s3", "bucket": "pg-extensions", "region": "eu-central-1", "secret": {"name": "aws-s3-secret"}} logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | ' /tmp/kuttl/pg/major-upgrade-15-to-16/cr.yaml logger.go:42: 19:21:32 | major-upgrade-15-to-16/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-15-to-16/cr.yaml logger.go:42: 19:21:33 | major-upgrade-15-to-16/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 created logger.go:42: 19:22:34 | major-upgrade-15-to-16/1-create-cluster | test step completed 1-create-cluster logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | starting test step 2-write-data logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:$(get_psql_user_pass major-upgrade-15-to-16-pguser-postgres)@$(get_psql_user_host major-upgrade-15-to-16-pguser-postgres)" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:$(get_psql_user_pass major-upgrade-15-to-16-pguser-postgres)@$(get_psql_user_host major-upgrade-15-to-16-pguser-postgres)" run_psql_local \ '\c postgres \\\ CREATE EXTENSION pg_cron' \ "postgres:$(get_psql_user_pass major-upgrade-15-to-16-pguser-postgres)@$(get_psql_user_host major-upgrade-15-to-16-pguser-postgres)"] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | + source ../../functions logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ realpath ../../.. logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ pwd logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-15-to-16 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ test_name=major-upgrade-15-to-16 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ command -v oc logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ -n '' ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export PG_VER=18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ PG_VER=18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ PGOV1_VER=14 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ printenv logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ which gdate logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ which date logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ date=/usr/sbin/date logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ which gsed logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++++ which sed logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ oc get projects logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ get_psql_user_pass major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ get_psql_user_host major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:34 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:22:35 | major-upgrade-15-to-16/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:22:35 | major-upgrade-15-to-16/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 19:22:35 | major-upgrade-15-to-16/2-write-data | + local uri=postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:22:35 | major-upgrade-15-to-16/2-write-data | + local driver=postgres logger.go:42: 19:22:35 | major-upgrade-15-to-16/2-write-data | ++ get_client_pod logger.go:42: 19:22:35 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:22:35 | major-upgrade-15-to-16/2-write-data | + kubectl -n kuttl-test-equipped-lab exec pg-client-65d98588cc-qqkfq -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc'\''' logger.go:42: 19:22:36 | major-upgrade-15-to-16/2-write-data | ++ get_psql_user_pass major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:36 | major-upgrade-15-to-16/2-write-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:36 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | ++ get_psql_user_host major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | + local uri=postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | + local driver=postgres logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | ++ get_client_pod logger.go:42: 19:22:37 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:22:38 | major-upgrade-15-to-16/2-write-data | + kubectl -n kuttl-test-equipped-lab exec pg-client-65d98588cc-qqkfq -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc'\''' logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | ++ get_psql_user_pass major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | ++ get_psql_user_host major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | + run_psql_local '\c postgres \\\ CREATE EXTENSION pg_cron' postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | + local 'command=\c postgres \\\ CREATE EXTENSION pg_cron' logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | + local uri=postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | + local driver=postgres logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | ++ get_client_pod logger.go:42: 19:22:39 | major-upgrade-15-to-16/2-write-data | ++ kubectl -n kuttl-test-equipped-lab get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:22:40 | major-upgrade-15-to-16/2-write-data | + kubectl -n kuttl-test-equipped-lab exec pg-client-65d98588cc-qqkfq -- bash -c 'printf '\''\c postgres \\\ CREATE EXTENSION pg_cron\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc'\''' logger.go:42: 19:22:41 | major-upgrade-15-to-16/2-write-data | test step completed 2-write-data logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | starting test step 3-read-from-primary logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-15-to-16-pguser-postgres)@$(get_psql_user_host major-upgrade-15-to-16-pguser-postgres)") kubectl create configmap -n "${NAMESPACE}" 03-read-from-primary --from-literal=data="${data}"] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | + source ../../functions logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ realpath ../../.. logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ pwd logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-15-to-16 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++ test_name=major-upgrade-15-to-16 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ command -v oc logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ ! -n '' ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ -n '' ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export PG_VER=18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ PG_VER=18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export BUCKET=pg-operator-testing logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ BUCKET=pg-operator-testing logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export PGOV1_VER=14 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ PGOV1_VER=14 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ MINIO_VER=5.4.0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ REGISTRY_NAME=docker.io logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ printenv logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ grep -E '^IMAGE' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ awk -F= '{print $1}' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ which gdate logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ which date logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ date=/usr/sbin/date logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ which gsed logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++++ which sed logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ sed=/usr/sbin/sed logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | ++ oc get projects logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ get_psql_user_pass major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ get_psql_user_host major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:22:41 | major-upgrade-15-to-16/3-read-from-primary | +++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:22:42 | major-upgrade-15-to-16/3-read-from-primary | ++ run_psql_local '\c myapp \\\ SELECT * from myApp;' postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:22:42 | major-upgrade-15-to-16/3-read-from-primary | ++ local 'command=\c myapp \\\ SELECT * from myApp;' logger.go:42: 19:22:42 | major-upgrade-15-to-16/3-read-from-primary | ++ local uri=postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:22:42 | major-upgrade-15-to-16/3-read-from-primary | ++ local driver=postgres logger.go:42: 19:22:42 | major-upgrade-15-to-16/3-read-from-primary | +++ get_client_pod logger.go:42: 19:22:42 | major-upgrade-15-to-16/3-read-from-primary | +++ kubectl -n kuttl-test-equipped-lab get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:22:42 | major-upgrade-15-to-16/3-read-from-primary | ++ kubectl -n kuttl-test-equipped-lab exec pg-client-65d98588cc-qqkfq -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc'\''' logger.go:42: 19:22:43 | major-upgrade-15-to-16/3-read-from-primary | + data=' 100500' logger.go:42: 19:22:43 | major-upgrade-15-to-16/3-read-from-primary | + kubectl create configmap -n kuttl-test-equipped-lab 03-read-from-primary '--from-literal=data= 100500' logger.go:42: 19:22:44 | major-upgrade-15-to-16/3-read-from-primary | configmap/03-read-from-primary created logger.go:42: 19:22:44 | major-upgrade-15-to-16/3-read-from-primary | test step completed 3-read-from-primary logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | starting test step 4-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c set -o errexit set -o xtrace source ../../functions kubectl apply -n ${NAMESPACE} -f - <<-EOF apiVersion: pgv2.percona.com/v2 kind: PerconaPGUpgrade metadata: name: 15-to-16 spec: postgresClusterName: major-upgrade-15-to-16 image: ${IMAGE_UPGRADE} fromPostgresVersion: 15 toPostgresVersion: 16 toPostgresImage: $(get_container_image "postgres" 16) toPgBouncerImage: $(get_container_image "pgbouncer" 16) toPgBackRestImage: $(get_container_image "pgbackrest" 16) EOF] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | + source ../../functions logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ realpath ../../.. logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ pwd logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-15-to-16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ test_name=major-upgrade-15-to-16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ command -v oc logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ ! -n '' ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ -n '' ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export PG_VER=18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ PG_VER=18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export BUCKET=pg-operator-testing logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ BUCKET=pg-operator-testing logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export PGOV1_VER=14 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ PGOV1_VER=14 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export MINIO_VER=5.4.0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ MINIO_VER=5.4.0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ REGISTRY_NAME=docker.io logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ printenv logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ grep -E '^IMAGE' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ awk -F= '{print $1}' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ which gdate logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ which date logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ date=/usr/sbin/date logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ which gsed logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++++ which sed logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | +++ sed=/usr/sbin/sed logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ oc get projects logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | + kubectl apply -n kuttl-test-equipped-lab -f - logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ get_container_image postgres 16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local component=postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local pgVersion=16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ operatorVersion=main logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ [[ postgres == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ [[ postgres == \p\g\b\a\c\k\r\e\s\t ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ get_container_image pgbouncer 16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local component=pgbouncer logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local pgVersion=16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ operatorVersion=main logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ [[ pgbouncer == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ get_container_image pgbackrest 16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local component=pgbackrest logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local pgVersion=16 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ operatorVersion=main logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ [[ pgbackrest == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ [[ pgbackrest == \p\g\b\a\c\k\r\e\s\t ]] logger.go:42: 19:22:44 | major-upgrade-15-to-16/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16 logger.go:42: 19:22:46 | major-upgrade-15-to-16/4-upgrade | perconapgupgrade.pgv2.percona.com/15-to-16 created logger.go:42: 19:22:46 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc ready 3 3 74s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 46s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-gfw2-0 4/4 Running 0 73s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-jcr4-0 4/4 Running 0 73s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 73s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb 2/2 Running 0 72s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj 2/2 Running 0 72s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k 2/2 Running 0 72s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 72s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 80s logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:47 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 46s logger.go:42: 19:22:53 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc stopping 3 3 81s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 53s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-gfw2-0 0/4 Completed 0 80s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-jcr4-0 0/4 Completed 0 80s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 80s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb 0/2 Completed 0 79s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj 2/2 Terminating 0 79s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k 2/2 Terminating 0 79s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 79s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 87s logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:54 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 53s logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc paused 0 0 88s logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 60s logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 86s logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 94s logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:01 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 60s logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc paused 0 0 95s logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 67s logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 101s logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:08 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 67s logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc paused 0 0 102s logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Init:0/5 0 4s logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 74s logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 108s logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Running 0/1 4s 4s logger.go:42: 19:23:15 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 74s logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc paused 0 0 109s logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Init:0/5 0 11s logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 81s logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 115s logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Running 0/1 11s 11s logger.go:42: 19:23:22 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 81s logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc paused 0 0 116s logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Init:4/5 0 18s logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 88s logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m2s logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Running 0/1 18s 18s logger.go:42: 19:23:29 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 88s logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc paused 0 0 2m3s logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 1/1 Running 0 25s logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 95s logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m9s logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Running 0/1 25s 25s logger.go:42: 19:23:36 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 95s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc paused 0 0 2m10s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Pending 0 3s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 ContainerCreating 0 3s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 32s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 102s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m16s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Running 0/1 3s 3s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Running 0/1 3s 3s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 32s logger.go:42: 19:23:43 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 102s logger.go:42: 19:23:50 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc paused 0 0 2m17s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 10s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 ContainerCreating 0 10s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 39s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-vpgb-plggw 0/1 Completed 0 109s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m23s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Running 0/1 10s 10s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Running 0/1 10s 10s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 39s logger.go:42: 19:23:51 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-vpgb Complete 1/1 30s 109s logger.go:42: 19:23:57 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc initializing 0 0 2m24s logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 17s logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 17s logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 46s logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m30s logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Complete 1/1 12s 18s logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Complete 1/1 14s 18s logger.go:42: 19:23:58 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 47s logger.go:42: 19:24:04 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc initializing 0 2 2m31s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 25s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 25s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 54s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 0/4 Init:0/5 0 8s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 6s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 7s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 7s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 0/2 Init:0/2 0 7s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m38s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Complete 1/1 12s 25s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Complete 1/1 14s 25s logger.go:42: 19:24:05 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 54s logger.go:42: 19:24:11 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc initializing 0 3 2m39s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 32s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 32s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 61s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 0/4 PodInitializing 0 15s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 13s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 14s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 14s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 14s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m45s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Complete 1/1 12s 32s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Complete 1/1 14s 32s logger.go:42: 19:24:12 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 61s logger.go:42: 19:24:18 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc ready 1 3 2m46s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 39s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 39s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 68s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-gfw2-0 0/4 Init:0/5 0 2s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-jcr4-0 0/4 Init:0/5 0 1s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 22s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 20s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 21s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 21s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 21s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m52s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Complete 1/1 12s 39s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Complete 1/1 14s 39s logger.go:42: 19:24:19 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 68s logger.go:42: 19:24:26 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc initializing 1 3 2m53s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 46s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 46s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 75s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-dpvw-nzsk7 1/1 Running 0 6s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-gfw2-0 0/4 Init:0/5 0 9s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-jcr4-0 0/4 Init:0/5 0 8s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 29s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 27s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 28s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 28s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 28s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 2m59s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Complete 1/1 12s 47s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Complete 1/1 14s 47s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 76s logger.go:42: 19:24:27 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-dpvw Running 0/1 7s 7s logger.go:42: 19:24:33 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc initializing 1 3 3m logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 54s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 54s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 83s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-dpvw-nzsk7 0/1 Completed 0 14s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-gfw2-0 0/4 PodInitializing 0 17s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-jcr4-0 0/4 PodInitializing 0 16s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 37s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 35s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 36s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 36s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 36s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 3m7s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Complete 1/1 12s 54s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Complete 1/1 14s 54s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 83s logger.go:42: 19:24:34 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-dpvw Complete 1/1 14s 14s logger.go:42: 19:24:40 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc ready 3 3 3m8s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 61s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 61s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 90s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-dpvw-nzsk7 0/1 Completed 0 21s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-gfw2-0 4/4 Running 0 24s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-jcr4-0 4/4 Running 0 23s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 44s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 42s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 43s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 43s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 43s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 3m14s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Complete 1/1 12s 61s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Complete 1/1 14s 61s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 90s logger.go:42: 19:24:41 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-dpvw Complete 1/1 14s 21s logger.go:42: 19:24:47 | major-upgrade-15-to-16/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc ready 3 3 3m15s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 68s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 68s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/15-to-16-pgdata-9rhjd 0/1 Completed 0 97s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-backup-dpvw-nzsk7 0/1 Completed 0 28s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-gfw2-0 4/4 Running 0 31s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-jcr4-0 4/4 Running 0 30s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 51s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 49s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 50s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 50s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/major-upgrade-15-to-16-repo-host-0 2/2 Running 0 50s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | pod/pg-client-65d98588cc-qqkfq 1/1 Running 0 3m21s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-gfw2 Complete 1/1 12s 68s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-major-upgrade-15-to-16-instance1-jcr4 Complete 1/1 14s 68s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | job.batch/15-to-16-pgdata Complete 1/1 29s 97s logger.go:42: 19:24:48 | major-upgrade-15-to-16/4-upgrade | job.batch/major-upgrade-15-to-16-backup-dpvw Complete 1/1 14s 28s logger.go:42: 19:24:53 | major-upgrade-15-to-16/4-upgrade | test step completed 4-upgrade logger.go:42: 19:24:53 | major-upgrade-15-to-16/5-change-repo | starting test step 5-change-repo logger.go:42: 19:24:53 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c set -o errexit set -o xtrace kubectl patch \ -n $NAMESPACE \ perconapgcluster major-upgrade-15-to-16 \ --type='json' \ -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]'] logger.go:42: 19:24:53 | major-upgrade-15-to-16/5-change-repo | + kubectl patch -n kuttl-test-equipped-lab perconapgcluster major-upgrade-15-to-16 --type=json '-p=[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]' logger.go:42: 19:24:54 | major-upgrade-15-to-16/5-change-repo | perconapgcluster.pgv2.percona.com/major-upgrade-15-to-16 patched logger.go:42: 19:24:55 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:24:55 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:24:55 | major-upgrade-15-to-16/5-change-repo | name: repo1 logger.go:42: 19:24:55 | major-upgrade-15-to-16/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:24:55 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:24:55 | major-upgrade-15-to-16/5-change-repo | volume: pvc-ff1c4cb0-8e5b-40c3-b2c7-28bc4e8b7861 logger.go:42: 19:24:56 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:24:57 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:24:57 | major-upgrade-15-to-16/5-change-repo | name: repo1 logger.go:42: 19:24:57 | major-upgrade-15-to-16/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:24:57 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:24:57 | major-upgrade-15-to-16/5-change-repo | volume: pvc-ff1c4cb0-8e5b-40c3-b2c7-28bc4e8b7861 logger.go:42: 19:24:58 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:24:59 | major-upgrade-15-to-16/5-change-repo | - name: repo2 logger.go:42: 19:24:59 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:00 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:00 | major-upgrade-15-to-16/5-change-repo | - name: repo2 logger.go:42: 19:25:00 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:02 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:02 | major-upgrade-15-to-16/5-change-repo | - name: repo2 logger.go:42: 19:25:02 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:03 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:04 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:04 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:04 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:04 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:05 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:05 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:05 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:05 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:05 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:07 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:07 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:07 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:07 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:07 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:08 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:09 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:09 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:09 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:09 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:10 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:11 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:11 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:11 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:11 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:12 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:12 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:12 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:12 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:12 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:13 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:14 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:14 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:14 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:14 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:15 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:16 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:16 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:16 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:16 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:17 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:17 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:17 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:17 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:17 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:19 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:19 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:19 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:19 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:19 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:20 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:21 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:21 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:21 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:21 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:22 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:22 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:22 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:22 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:22 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:24 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:24 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:24 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:24 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:24 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:25 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:26 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:26 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:26 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:26 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:27 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:28 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:28 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:28 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:28 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:29 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:29 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:29 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:29 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:29 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:30 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:31 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:31 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:31 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:31 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:32 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:33 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:33 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:33 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:33 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:34 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:34 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:34 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:34 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:34 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:36 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:36 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:36 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:36 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:36 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:37 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:38 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:38 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:38 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:38 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:39 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:39 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:39 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:39 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:39 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:41 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:41 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:41 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:41 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:41 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:42 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:43 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:43 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:43 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:43 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:44 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:45 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:45 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:45 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:45 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:46 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:46 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:46 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:46 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:46 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:47 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:48 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:48 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:48 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:48 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:49 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:50 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:50 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:50 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:50 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:51 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:51 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:51 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:51 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:51 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:53 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:53 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:53 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:53 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:53 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:54 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:55 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:55 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:55 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:55 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:56 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:56 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:56 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:56 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:56 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:58 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:25:58 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:25:58 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:25:58 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:25:58 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:25:59 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:00 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:00 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:00 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:00 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:01 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:02 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:02 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:02 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:02 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:03 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:03 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:03 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:03 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:03 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:04 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:05 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:05 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:05 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:05 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:06 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:07 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:07 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:07 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:07 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:08 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:08 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:08 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:08 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:08 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:09 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:10 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:10 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:10 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:10 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:11 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:12 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:12 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:12 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:12 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:13 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:13 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:13 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:13 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:13 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:15 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:15 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:15 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:15 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:15 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:16 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:17 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:17 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:17 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:17 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:18 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:19 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:19 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:19 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:19 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:20 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:20 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:20 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:20 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:20 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:21 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:22 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:22 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:22 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:22 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:23 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:24 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:24 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:24 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:24 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:25 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:25 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:25 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:25 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:25 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:26 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:27 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:27 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:27 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:27 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:28 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:29 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:29 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:29 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: false logger.go:42: 19:26:29 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:30 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:30 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:30 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:30 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:30 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:32 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:32 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:32 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:32 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:32 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:33 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:34 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:34 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:34 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:34 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:35 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:35 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:35 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:35 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:35 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:37 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:37 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:37 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:37 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:37 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:38 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:39 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:39 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:39 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:39 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:40 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:41 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:41 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:41 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:41 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:42 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:42 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:42 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:42 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:42 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:43 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:44 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:44 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:44 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:44 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:45 | major-upgrade-15-to-16/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-15-to-16 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:26:46 | major-upgrade-15-to-16/5-change-repo | - bound: true logger.go:42: 19:26:46 | major-upgrade-15-to-16/5-change-repo | name: repo2 logger.go:42: 19:26:46 | major-upgrade-15-to-16/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:26:46 | major-upgrade-15-to-16/5-change-repo | stanzaCreated: true logger.go:42: 19:26:46 | major-upgrade-15-to-16/5-change-repo | volume: pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 logger.go:42: 19:26:46 | major-upgrade-15-to-16/5-change-repo | test step completed 5-change-repo logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | starting test step 6-write-data logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' \ "postgres:$(get_psql_user_pass major-upgrade-15-to-16-pguser-postgres)@$(get_psql_user_host major-upgrade-15-to-16-pguser-postgres)"] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | + source ../../functions logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ realpath ../../.. logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ pwd logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-15-to-16 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ test_name=major-upgrade-15-to-16 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ command -v oc logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ ! -n '' ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ -n '' ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export PG_VER=18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ PG_VER=18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export PGOV1_VER=14 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ PGOV1_VER=14 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ printenv logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ grep -E '^IMAGE' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ awk -F= '{print $1}' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ which gdate logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ which date logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ date=/usr/sbin/date logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ which gsed logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++++ which sed logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | +++ sed=/usr/sbin/sed logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ oc get projects logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ get_psql_user_pass major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ get_psql_user_host major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:26:46 | major-upgrade-15-to-16/6-write-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:26:47 | major-upgrade-15-to-16/6-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:26:47 | major-upgrade-15-to-16/6-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' logger.go:42: 19:26:47 | major-upgrade-15-to-16/6-write-data | + local uri=postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:26:47 | major-upgrade-15-to-16/6-write-data | + local driver=postgres logger.go:42: 19:26:47 | major-upgrade-15-to-16/6-write-data | ++ get_client_pod logger.go:42: 19:26:47 | major-upgrade-15-to-16/6-write-data | ++ kubectl -n kuttl-test-equipped-lab get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:26:47 | major-upgrade-15-to-16/6-write-data | + kubectl -n kuttl-test-equipped-lab exec pg-client-65d98588cc-qqkfq -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc'\''' logger.go:42: 19:26:48 | major-upgrade-15-to-16/6-write-data | test step completed 6-write-data logger.go:42: 19:26:48 | major-upgrade-15-to-16/7-run-backup | starting test step 7-run-backup logger.go:42: 19:26:49 | major-upgrade-15-to-16/7-run-backup | PerconaPGBackup:kuttl-test-equipped-lab/backup-after-15-to-16 created logger.go:42: 19:27:17 | major-upgrade-15-to-16/7-run-backup | test step completed 7-run-backup logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | starting test step 8-read-from-primary logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-15-to-16-pguser-postgres)@$(get_psql_user_host major-upgrade-15-to-16-pguser-postgres)") kubectl create configmap -n "${NAMESPACE}" 08-read-from-primary --from-literal=data="${data}"] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | + source ../../functions logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ realpath ../../.. logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ pwd logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-15-to-16 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++ test_name=major-upgrade-15-to-16 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ command -v oc logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ ! -n '' ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ -n '' ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export PG_VER=18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ PG_VER=18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export BUCKET=pg-operator-testing logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ BUCKET=pg-operator-testing logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export PGOV1_VER=14 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ PGOV1_VER=14 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ MINIO_VER=5.4.0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ REGISTRY_NAME=docker.io logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ printenv logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ grep -E '^IMAGE' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ awk -F= '{print $1}' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ which gdate logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ which date logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ date=/usr/sbin/date logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ which gsed logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++++ which sed logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ sed=/usr/sbin/sed logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | ++ oc get projects logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ get_psql_user_pass major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:27:17 | major-upgrade-15-to-16/8-read-from-primary | +++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | +++ get_psql_user_host major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | +++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | +++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | ++ run_psql_local '\c myapp \\\ SELECT * from myApp;' postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | ++ local 'command=\c myapp \\\ SELECT * from myApp;' logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | ++ local uri=postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | ++ local driver=postgres logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | +++ get_client_pod logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | +++ kubectl -n kuttl-test-equipped-lab get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:27:18 | major-upgrade-15-to-16/8-read-from-primary | ++ kubectl -n kuttl-test-equipped-lab exec pg-client-65d98588cc-qqkfq -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc'\''' logger.go:42: 19:27:19 | major-upgrade-15-to-16/8-read-from-primary | + data=' 100500 logger.go:42: 19:27:19 | major-upgrade-15-to-16/8-read-from-primary | 100501' logger.go:42: 19:27:19 | major-upgrade-15-to-16/8-read-from-primary | + kubectl create configmap -n kuttl-test-equipped-lab 08-read-from-primary '--from-literal=data= 100500 logger.go:42: 19:27:19 | major-upgrade-15-to-16/8-read-from-primary | 100501' logger.go:42: 19:27:20 | major-upgrade-15-to-16/8-read-from-primary | configmap/08-read-from-primary created logger.go:42: 19:27:20 | major-upgrade-15-to-16/8-read-from-primary | test step completed 8-read-from-primary logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | starting test step 9-delete-data logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ '\c myapp \\\ TRUNCATE TABLE myApp' \ "postgres:$(get_psql_user_pass major-upgrade-15-to-16-pguser-postgres)@$(get_psql_user_host major-upgrade-15-to-16-pguser-postgres)"] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | + source ../../functions logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ realpath ../../.. logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ pwd logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-15-to-16 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++ test_name=major-upgrade-15-to-16 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-15-to-16 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ command -v oc logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ ! -n '' ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ -n '' ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export PG_VER=18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ PG_VER=18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export PGOV1_VER=14 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ PGOV1_VER=14 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ printenv logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ grep -E '^IMAGE' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ awk -F= '{print $1}' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ which gdate logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ which date logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ date=/usr/sbin/date logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ which gsed logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++++ which sed logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | +++ sed=/usr/sbin/sed logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++ oc get projects logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++ get_psql_user_pass major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:27:20 | major-upgrade-15-to-16/9-delete-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | ++ get_psql_user_host major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | ++ local secret_name=major-upgrade-15-to-16-pguser-postgres logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | ++ kubectl -n kuttl-test-equipped-lab get secret/major-upgrade-15-to-16-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | + run_psql_local '\c myapp \\\ TRUNCATE TABLE myApp' postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | + local 'command=\c myapp \\\ TRUNCATE TABLE myApp' logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | + local uri=postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | + local driver=postgres logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | ++ get_client_pod logger.go:42: 19:27:21 | major-upgrade-15-to-16/9-delete-data | ++ kubectl -n kuttl-test-equipped-lab get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:27:22 | major-upgrade-15-to-16/9-delete-data | + kubectl -n kuttl-test-equipped-lab exec pg-client-65d98588cc-qqkfq -- bash -c 'printf '\''\c myapp \\\ TRUNCATE TABLE myApp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:h5d9MguJdps4p19TGzXMHQe4@major-upgrade-15-to-16-primary.kuttl-test-equipped-lab.svc'\''' logger.go:42: 19:27:23 | major-upgrade-15-to-16/9-delete-data | test step completed 9-delete-data logger.go:42: 19:27:23 | major-upgrade-15-to-16/10-run-restore | starting test step 10-run-restore logger.go:42: 19:27:23 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-15-to-16 primary name) latest_full_repo2_backup=$(kubectl -n ${NAMESPACE} exec ${primary} -- pgbackrest info --output json --log-level-console=info | jq -r '[.[] | .backup[] | select(.type == "full") | select(.database["repo-key"] == 2)][-1].label') cat </dev/null done sleep 30] logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 3m47s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 3m47s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 4m16s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 37s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 58s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 4/4 Running 0 3m10s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 4/4 Running 0 3m9s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 3m30s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 3m28s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 3m29s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 3m29s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 2m30s logger.go:42: 19:27:27 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 6m logger.go:42: 19:27:29 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:27:31 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 3m52s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 3m52s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 4m21s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 42s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 63s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbackrest-restore-z74bv 0/1 Init:0/1 0 1s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 3m33s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 3m34s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 3m34s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 2m35s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 6m5s logger.go:42: 19:27:32 | major-upgrade-15-to-16/10-run-restore | No resources found in kuttl-test-equipped-lab namespace. logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 4m24s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 4m24s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 4m53s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 74s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 95s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbackrest-restore-z74bv 0/1 Completed 0 33s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 4m5s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 4m6s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 4m6s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 3m7s logger.go:42: 19:28:04 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 6m37s logger.go:42: 19:28:06 | major-upgrade-15-to-16/10-run-restore | Waiting for major-upgrade-15-to-16-instance1-pzj7-0 to start running logger.go:42: 19:28:37 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 4m57s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 4m57s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 5m26s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 107s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 2m8s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 19s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 19s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 33s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 4m38s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 4m39s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 4m39s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 3m40s logger.go:42: 19:28:38 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 7m10s logger.go:42: 19:28:39 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:36.366 UTC [133] LOG: completed backup recovery with redo LSN 0/C000028 and end LSN 0/C000100 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:36.366 UTC [133] LOG: consistent recovery state reached at 0/C000100 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:36.366 UTC [128] LOG: database system is ready to accept read-only connections logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:36.505 UTC [161] LOG: started streaming WAL from primary at 0/E000000 on timeline 2 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.041 UTC [128] LOG: received fast shutdown request logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.044 UTC [128] LOG: aborting any active transactions logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.044 UTC [160] FATAL: terminating connection due to administrator command logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.044 UTC [196] FATAL: terminating connection due to administrator command logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.047 UTC [161] FATAL: terminating walreceiver process due to administrator command logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.049 UTC [131] LOG: shutting down logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.049 UTC [131] LOG: restartpoint starting: shutdown immediate logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.074 UTC [131] LOG: restartpoint complete: wrote 44 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.010 s, sync=0.007 s, total=0.025 s; sync files=21, longest=0.004 s, average=0.001 s; distance=98304 kB, estimate=98304 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.074 UTC [131] LOG: recovery restart point at 0/12000028 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.074 UTC [131] DETAIL: Last completed transaction was at log time 2026-02-20 19:27:23.035873+00. logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.076 UTC [128] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.126 UTC [128] LOG: database system is shut down logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:35.995 UTC [141] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:35.996 UTC [141] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:35.996 UTC [141] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.021 UTC [141] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.038 UTC [146] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.069 UTC [146] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.098 UTC [146] LOG: entering standby mode logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.128 UTC [146] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.455 UTC [146] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.503 UTC [146] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.505 UTC [141] LOG: startup process (PID 146) exited with exit code 1 logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.505 UTC [141] LOG: aborting startup due to startup process failure logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.505 UTC [141] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:36.517 UTC [141] LOG: database system is shut down logger.go:42: 19:28:41 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:37.172 UTC [133] LOG: completed backup recovery with redo LSN 0/D000028 and end LSN 0/D000100 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:37.172 UTC [133] LOG: consistent recovery state reached at 0/D000100 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:37.172 UTC [128] LOG: database system is ready to accept read-only connections logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:37.305 UTC [168] LOG: started streaming WAL from primary at 0/E000000 on timeline 2 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:28.975 UTC [128] LOG: received fast shutdown request logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:28.978 UTC [128] LOG: aborting any active transactions logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:28.978 UTC [180] FATAL: terminating connection due to administrator command logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:28.978 UTC [167] FATAL: terminating connection due to administrator command logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:28.980 UTC [168] FATAL: terminating walreceiver process due to administrator command logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:28.982 UTC [131] LOG: shutting down logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:28.982 UTC [131] LOG: restartpoint starting: shutdown immediate logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.004 UTC [131] LOG: restartpoint complete: wrote 44 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.008 s, sync=0.006 s, total=0.022 s; sync files=21, longest=0.003 s, average=0.001 s; distance=81920 kB, estimate=81920 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.004 UTC [131] LOG: recovery restart point at 0/12000028 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.004 UTC [131] DETAIL: Last completed transaction was at log time 2026-02-20 19:27:23.035873+00. logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.006 UTC [128] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:27:29.053 UTC [128] LOG: database system is shut down logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.003 UTC [141] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.003 UTC [141] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.003 UTC [141] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.028 UTC [141] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.043 UTC [146] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.069 UTC [146] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.096 UTC [146] LOG: entering standby mode logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.122 UTC [146] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.454 UTC [146] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.486 UTC [146] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.487 UTC [141] LOG: startup process (PID 146) exited with exit code 1 logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.487 UTC [141] LOG: aborting startup due to startup process failure logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.487 UTC [141] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:37.495 UTC [141] LOG: database system is shut down logger.go:42: 19:28:43 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:15.751 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 16, system-id 7609030249273004126 do not match repo1 stanza version 15, system-id 7609029839975215230 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:28:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:29:15 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 5m35s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 5m35s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 6m4s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 2m25s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 2m46s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 57s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 57s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 71s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 5m16s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 5m17s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 5m17s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 4m18s logger.go:42: 19:29:16 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 7m48s logger.go:42: 19:29:17 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:48.022 UTC [217] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:48.033 UTC [217] LOG: database system is shut down logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.451 UTC [286] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.452 UTC [286] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.452 UTC [286] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.477 UTC [286] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.486 UTC [291] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.517 UTC [291] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.548 UTC [291] LOG: entering standby mode logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.579 UTC [291] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.904 UTC [291] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.940 UTC [291] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.942 UTC [286] LOG: startup process (PID 291) exited with exit code 1 logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.942 UTC [286] LOG: aborting startup due to startup process failure logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.942 UTC [286] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.953 UTC [286] LOG: database system is shut down logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.452 UTC [330] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.453 UTC [330] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.453 UTC [330] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.479 UTC [330] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.487 UTC [335] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.518 UTC [335] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.549 UTC [335] LOG: entering standby mode logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.578 UTC [335] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.910 UTC [335] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.946 UTC [335] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.947 UTC [330] LOG: startup process (PID 335) exited with exit code 1 logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.947 UTC [330] LOG: aborting startup due to startup process failure logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.947 UTC [330] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:18 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.959 UTC [330] LOG: database system is shut down logger.go:42: 19:29:19 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:47.861 UTC [193] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:47.869 UTC [193] LOG: database system is shut down logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.305 UTC [262] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.306 UTC [262] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.306 UTC [262] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.331 UTC [262] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.339 UTC [267] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.365 UTC [267] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.391 UTC [267] LOG: entering standby mode logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.417 UTC [267] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.738 UTC [267] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.772 UTC [267] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.773 UTC [262] LOG: startup process (PID 267) exited with exit code 1 logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.773 UTC [262] LOG: aborting startup due to startup process failure logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.773 UTC [262] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.781 UTC [262] LOG: database system is shut down logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.327 UTC [306] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.327 UTC [306] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.327 UTC [306] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.353 UTC [306] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.362 UTC [311] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.387 UTC [311] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.415 UTC [311] LOG: entering standby mode logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.440 UTC [311] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.562 UTC [311] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.603 UTC [311] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.604 UTC [306] LOG: startup process (PID 311) exited with exit code 1 logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.604 UTC [306] LOG: aborting startup due to startup process failure logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.604 UTC [306] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.616 UTC [306] LOG: database system is shut down logger.go:42: 19:29:21 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:15.751 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 16, system-id 7609030249273004126 do not match repo1 stanza version 15, system-id 7609029839975215230 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:29:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:29:53 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 6m14s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 6m14s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 6m43s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 3m4s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 3m25s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 96s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 96s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 110s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 5m55s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 5m56s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 5m56s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 4m57s logger.go:42: 19:29:54 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 8m27s logger.go:42: 19:29:56 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.942 UTC [286] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:07.953 UTC [286] LOG: database system is shut down logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.452 UTC [330] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.453 UTC [330] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.453 UTC [330] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.479 UTC [330] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.487 UTC [335] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.518 UTC [335] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.549 UTC [335] LOG: entering standby mode logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.578 UTC [335] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.910 UTC [335] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.946 UTC [335] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.947 UTC [330] LOG: startup process (PID 335) exited with exit code 1 logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.947 UTC [330] LOG: aborting startup due to startup process failure logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.947 UTC [330] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.959 UTC [330] LOG: database system is shut down logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.417 UTC [413] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.418 UTC [413] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.418 UTC [413] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.443 UTC [413] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.452 UTC [418] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.483 UTC [418] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.512 UTC [418] LOG: entering standby mode logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.544 UTC [418] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.867 UTC [418] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.902 UTC [418] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.904 UTC [413] LOG: startup process (PID 418) exited with exit code 1 logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.904 UTC [413] LOG: aborting startup due to startup process failure logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.904 UTC [413] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.915 UTC [413] LOG: database system is shut down logger.go:42: 19:29:58 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.604 UTC [306] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:17.616 UTC [306] LOG: database system is shut down logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.345 UTC [383] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.346 UTC [383] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.346 UTC [383] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.374 UTC [383] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.383 UTC [388] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.410 UTC [388] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.439 UTC [388] LOG: entering standby mode logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.468 UTC [388] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.791 UTC [388] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.825 UTC [388] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.826 UTC [383] LOG: startup process (PID 388) exited with exit code 1 logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.826 UTC [383] LOG: aborting startup due to startup process failure logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.826 UTC [383] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:37.834 UTC [383] LOG: database system is shut down logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.366 UTC [453] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.367 UTC [453] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.367 UTC [453] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.394 UTC [453] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.404 UTC [458] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.434 UTC [458] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.466 UTC [458] LOG: entering standby mode logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.500 UTC [458] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.828 UTC [458] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.863 UTC [458] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.864 UTC [453] LOG: startup process (PID 458) exited with exit code 1 logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.864 UTC [453] LOG: aborting startup due to startup process failure logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.865 UTC [453] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.872 UTC [453] LOG: database system is shut down logger.go:42: 19:29:59 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:15.751 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 16, system-id 7609030249273004126 do not match repo1 stanza version 15, system-id 7609029839975215230 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:30:01 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:30:32 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 6m52s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 6m52s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 7m21s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 3m42s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 4m3s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 2m14s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 2m14s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 2m28s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 6m33s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 6m34s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 6m34s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 5m35s logger.go:42: 19:30:33 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 9m5s logger.go:42: 19:30:34 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.932 UTC [483] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.944 UTC [483] LOG: database system is shut down logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.415 UTC [535] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.416 UTC [535] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.416 UTC [535] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.441 UTC [535] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.450 UTC [540] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.483 UTC [540] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.512 UTC [540] LOG: entering standby mode logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.545 UTC [540] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.868 UTC [540] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.903 UTC [540] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.905 UTC [535] LOG: startup process (PID 540) exited with exit code 1 logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.905 UTC [535] LOG: aborting startup due to startup process failure logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.905 UTC [535] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.916 UTC [535] LOG: database system is shut down logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.411 UTC [604] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.412 UTC [604] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.412 UTC [604] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.437 UTC [604] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.446 UTC [609] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.478 UTC [609] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.507 UTC [609] LOG: entering standby mode logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.539 UTC [609] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.862 UTC [609] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.897 UTC [609] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.899 UTC [604] LOG: startup process (PID 609) exited with exit code 1 logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.899 UTC [604] LOG: aborting startup due to startup process failure logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.899 UTC [604] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.910 UTC [604] LOG: database system is shut down logger.go:42: 19:30:36 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.865 UTC [453] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:29:57.872 UTC [453] LOG: database system is shut down logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.400 UTC [505] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.401 UTC [505] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.401 UTC [505] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.428 UTC [505] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.440 UTC [510] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.472 UTC [510] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.499 UTC [510] LOG: entering standby mode logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.529 UTC [510] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.855 UTC [510] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.890 UTC [510] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.892 UTC [505] LOG: startup process (PID 510) exited with exit code 1 logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.892 UTC [505] LOG: aborting startup due to startup process failure logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.892 UTC [505] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:07.900 UTC [505] LOG: database system is shut down logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.346 UTC [571] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.346 UTC [571] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.346 UTC [571] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.374 UTC [571] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.384 UTC [576] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.414 UTC [576] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.441 UTC [576] LOG: entering standby mode logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.473 UTC [576] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.796 UTC [576] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.831 UTC [576] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.833 UTC [571] LOG: startup process (PID 576) exited with exit code 1 logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.833 UTC [571] LOG: aborting startup due to startup process failure logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.833 UTC [571] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.841 UTC [571] LOG: database system is shut down logger.go:42: 19:30:38 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:15.751 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 16, system-id 7609030249273004126 do not match repo1 stanza version 15, system-id 7609029839975215230 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:30:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:31:10 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 7m31s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 7m31s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 8m logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 4m21s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 4m42s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 2m53s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 2m53s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 3m7s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 7m12s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 7m13s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 7m13s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 6m14s logger.go:42: 19:31:11 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 9m44s logger.go:42: 19:31:12 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.899 UTC [604] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.910 UTC [604] LOG: database system is shut down logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.412 UTC [656] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.412 UTC [656] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.412 UTC [656] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.438 UTC [656] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.447 UTC [661] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.478 UTC [661] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.508 UTC [661] LOG: entering standby mode logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.541 UTC [661] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.864 UTC [661] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.899 UTC [661] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.901 UTC [656] LOG: startup process (PID 661) exited with exit code 1 logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.901 UTC [656] LOG: aborting startup due to startup process failure logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.901 UTC [656] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:37.912 UTC [656] LOG: database system is shut down logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.430 UTC [724] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.431 UTC [724] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.431 UTC [724] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.457 UTC [724] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.466 UTC [729] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.499 UTC [729] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.529 UTC [729] LOG: entering standby mode logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.559 UTC [729] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.883 UTC [729] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.917 UTC [729] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.919 UTC [724] LOG: startup process (PID 729) exited with exit code 1 logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.919 UTC [724] LOG: aborting startup due to startup process failure logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.919 UTC [724] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.930 UTC [724] LOG: database system is shut down logger.go:42: 19:31:14 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.833 UTC [571] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:27.841 UTC [571] LOG: database system is shut down logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.341 UTC [648] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.342 UTC [648] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.342 UTC [648] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.368 UTC [648] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.378 UTC [653] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.405 UTC [653] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.433 UTC [653] LOG: entering standby mode logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.459 UTC [653] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.782 UTC [653] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.818 UTC [653] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.819 UTC [648] LOG: startup process (PID 653) exited with exit code 1 logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.819 UTC [648] LOG: aborting startup due to startup process failure logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.819 UTC [648] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:47.827 UTC [648] LOG: database system is shut down logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.324 UTC [691] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.324 UTC [691] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.324 UTC [691] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.351 UTC [691] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.361 UTC [696] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.399 UTC [696] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.430 UTC [696] LOG: entering standby mode logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.460 UTC [696] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.784 UTC [696] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.819 UTC [696] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.820 UTC [691] LOG: startup process (PID 696) exited with exit code 1 logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.820 UTC [691] LOG: aborting startup due to startup process failure logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.820 UTC [691] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.828 UTC [691] LOG: database system is shut down logger.go:42: 19:31:16 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:15.751 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 16, system-id 7609030249273004126 do not match repo1 stanza version 15, system-id 7609029839975215230 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:31:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:31:48 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 8m9s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 8m9s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 8m38s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 4m59s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 5m20s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 3m31s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 3m31s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 3m45s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 7m50s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 7m51s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 7m51s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 6m52s logger.go:42: 19:31:49 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 10m logger.go:42: 19:31:51 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.919 UTC [724] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.930 UTC [724] LOG: database system is shut down logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.398 UTC [801] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.399 UTC [801] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.399 UTC [801] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.425 UTC [801] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.433 UTC [806] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.467 UTC [806] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.499 UTC [806] LOG: entering standby mode logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.529 UTC [806] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.852 UTC [806] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.890 UTC [806] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.892 UTC [801] LOG: startup process (PID 806) exited with exit code 1 logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.892 UTC [801] LOG: aborting startup due to startup process failure logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.892 UTC [801] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.905 UTC [801] LOG: database system is shut down logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.405 UTC [878] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.406 UTC [878] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.406 UTC [878] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.431 UTC [878] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.440 UTC [883] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.471 UTC [883] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.501 UTC [883] LOG: entering standby mode logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.531 UTC [883] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.854 UTC [883] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.890 UTC [883] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.891 UTC [878] LOG: startup process (PID 883) exited with exit code 1 logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.891 UTC [878] LOG: aborting startup due to startup process failure logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.891 UTC [878] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.903 UTC [878] LOG: database system is shut down logger.go:42: 19:31:52 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.820 UTC [691] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:30:57.828 UTC [691] LOG: database system is shut down logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.325 UTC [768] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.325 UTC [768] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.325 UTC [768] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.353 UTC [768] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.363 UTC [773] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.393 UTC [773] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.422 UTC [773] LOG: entering standby mode logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.451 UTC [773] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.775 UTC [773] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.811 UTC [773] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.812 UTC [768] LOG: startup process (PID 773) exited with exit code 1 logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.812 UTC [768] LOG: aborting startup due to startup process failure logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.812 UTC [768] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:17.820 UTC [768] LOG: database system is shut down logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.324 UTC [837] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.324 UTC [837] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.325 UTC [837] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.352 UTC [837] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.360 UTC [842] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.388 UTC [842] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.415 UTC [842] LOG: entering standby mode logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.443 UTC [842] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.767 UTC [842] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.802 UTC [842] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.803 UTC [837] LOG: startup process (PID 842) exited with exit code 1 logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.803 UTC [837] LOG: aborting startup due to startup process failure logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.803 UTC [837] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:37.811 UTC [837] LOG: database system is shut down logger.go:42: 19:31:54 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:15.751 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 16, system-id 7609030249273004126 do not match repo1 stanza version 15, system-id 7609029839975215230 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:31:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:32:27 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 8m47s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 8m47s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 9m16s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 5m37s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 5m58s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 4m9s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 4m9s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 4m23s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 8m28s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 8m29s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 8m29s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 7m30s logger.go:42: 19:32:28 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 11m logger.go:42: 19:32:29 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:57.879 UTC [954] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:57.891 UTC [954] LOG: database system is shut down logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.418 UTC [1023] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.419 UTC [1023] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.419 UTC [1023] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.445 UTC [1023] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.453 UTC [1028] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.484 UTC [1028] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.511 UTC [1028] LOG: entering standby mode logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.540 UTC [1028] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.862 UTC [1028] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.897 UTC [1028] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.899 UTC [1023] LOG: startup process (PID 1028) exited with exit code 1 logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.899 UTC [1023] LOG: aborting startup due to startup process failure logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.899 UTC [1023] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.910 UTC [1023] LOG: database system is shut down logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.407 UTC [1068] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.408 UTC [1068] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.408 UTC [1068] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.433 UTC [1068] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.442 UTC [1073] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.474 UTC [1073] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.502 UTC [1073] LOG: entering standby mode logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.532 UTC [1073] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.856 UTC [1073] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.890 UTC [1073] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.892 UTC [1068] LOG: startup process (PID 1073) exited with exit code 1 logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.892 UTC [1068] LOG: aborting startup due to startup process failure logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.892 UTC [1068] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:30 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.904 UTC [1068] LOG: database system is shut down logger.go:42: 19:32:31 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:57.770 UTC [914] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:31:57.778 UTC [914] LOG: database system is shut down logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.296 UTC [983] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.296 UTC [983] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.296 UTC [983] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.323 UTC [983] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.332 UTC [988] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.360 UTC [988] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.386 UTC [988] LOG: entering standby mode logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.413 UTC [988] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.737 UTC [988] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.771 UTC [988] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.773 UTC [983] LOG: startup process (PID 988) exited with exit code 1 logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.773 UTC [983] LOG: aborting startup due to startup process failure logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.773 UTC [983] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:17.781 UTC [983] LOG: database system is shut down logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.329 UTC [1035] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.330 UTC [1035] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.330 UTC [1035] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.356 UTC [1035] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.365 UTC [1040] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.392 UTC [1040] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.417 UTC [1040] LOG: entering standby mode logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.444 UTC [1040] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.767 UTC [1040] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.800 UTC [1040] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.802 UTC [1035] LOG: startup process (PID 1040) exited with exit code 1 logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.802 UTC [1035] LOG: aborting startup due to startup process failure logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.802 UTC [1035] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.810 UTC [1035] LOG: database system is shut down logger.go:42: 19:32:33 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:15.751 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 16, system-id 7609030249273004126 do not match repo1 stanza version 15, system-id 7609029839975215230 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:32:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:33:05 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 9m25s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 9m25s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 9m54s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 6m15s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 6m36s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 4m47s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 4m47s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 5m1s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 9m6s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 9m7s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 9m7s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 8m8s logger.go:42: 19:33:06 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 11m logger.go:42: 19:33:07 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.892 UTC [1068] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:27.904 UTC [1068] LOG: database system is shut down logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.398 UTC [1119] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.399 UTC [1119] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.399 UTC [1119] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.424 UTC [1119] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.433 UTC [1124] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.464 UTC [1124] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.493 UTC [1124] LOG: entering standby mode logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.523 UTC [1124] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.846 UTC [1124] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.881 UTC [1124] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.882 UTC [1119] LOG: startup process (PID 1124) exited with exit code 1 logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.882 UTC [1119] LOG: aborting startup due to startup process failure logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.882 UTC [1119] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.893 UTC [1119] LOG: database system is shut down logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.409 UTC [1187] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.410 UTC [1187] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.410 UTC [1187] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.435 UTC [1187] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.444 UTC [1192] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.478 UTC [1192] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.506 UTC [1192] LOG: entering standby mode logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.536 UTC [1192] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.860 UTC [1192] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.895 UTC [1192] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.896 UTC [1187] LOG: startup process (PID 1192) exited with exit code 1 logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.896 UTC [1187] LOG: aborting startup due to startup process failure logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.896 UTC [1187] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.908 UTC [1187] LOG: database system is shut down logger.go:42: 19:33:09 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.791 UTC [1087] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:37.799 UTC [1087] LOG: database system is shut down logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.321 UTC [1156] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.321 UTC [1156] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.321 UTC [1156] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.347 UTC [1156] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.356 UTC [1161] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.384 UTC [1161] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.411 UTC [1161] LOG: entering standby mode logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.439 UTC [1161] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.762 UTC [1161] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.797 UTC [1161] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.799 UTC [1156] LOG: startup process (PID 1161) exited with exit code 1 logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.799 UTC [1156] LOG: aborting startup due to startup process failure logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.799 UTC [1156] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.806 UTC [1156] LOG: database system is shut down logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.313 UTC [1200] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.313 UTC [1200] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.313 UTC [1200] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.340 UTC [1200] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.348 UTC [1205] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.378 UTC [1205] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.403 UTC [1205] LOG: entering standby mode logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.430 UTC [1205] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.752 UTC [1205] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.786 UTC [1205] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.787 UTC [1200] LOG: startup process (PID 1205) exited with exit code 1 logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.787 UTC [1200] LOG: aborting startup due to startup process failure logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.787 UTC [1200] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.795 UTC [1200] LOG: database system is shut down logger.go:42: 19:33:11 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:15.751 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 16, system-id 7609030249273004126 do not match repo1 stanza version 15, system-id 7609029839975215230 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:33:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:33:43 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 10m logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 10m logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 10m logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 6m54s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 7m15s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 5m26s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 5m26s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 5m40s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 9m45s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 9m46s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 9m46s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 8m47s logger.go:42: 19:33:44 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 12m logger.go:42: 19:33:46 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.896 UTC [1187] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:32:57.908 UTC [1187] LOG: database system is shut down logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.397 UTC [1264] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.398 UTC [1264] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.398 UTC [1264] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.423 UTC [1264] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.432 UTC [1269] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.463 UTC [1269] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.491 UTC [1269] LOG: entering standby mode logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.520 UTC [1269] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.844 UTC [1269] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.879 UTC [1269] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.880 UTC [1264] LOG: startup process (PID 1269) exited with exit code 1 logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.880 UTC [1264] LOG: aborting startup due to startup process failure logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.880 UTC [1264] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:17.892 UTC [1264] LOG: database system is shut down logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.395 UTC [1308] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.396 UTC [1308] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.396 UTC [1308] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.422 UTC [1308] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.430 UTC [1313] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.462 UTC [1313] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.493 UTC [1313] LOG: entering standby mode logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.525 UTC [1313] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.848 UTC [1313] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.883 UTC [1313] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.885 UTC [1308] LOG: startup process (PID 1313) exited with exit code 1 logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.885 UTC [1308] LOG: aborting startup due to startup process failure logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.885 UTC [1308] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.896 UTC [1308] LOG: database system is shut down logger.go:42: 19:33:47 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.787 UTC [1200] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:07.795 UTC [1200] LOG: database system is shut down logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.298 UTC [1277] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.299 UTC [1277] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.299 UTC [1277] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.325 UTC [1277] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.333 UTC [1282] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.360 UTC [1282] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.387 UTC [1282] LOG: entering standby mode logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.413 UTC [1282] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.736 UTC [1282] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.770 UTC [1282] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.772 UTC [1277] LOG: startup process (PID 1282) exited with exit code 1 logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.772 UTC [1277] LOG: aborting startup due to startup process failure logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.772 UTC [1277] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:27.780 UTC [1277] LOG: database system is shut down logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.304 UTC [1346] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.305 UTC [1346] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.305 UTC [1346] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.331 UTC [1346] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.339 UTC [1351] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.365 UTC [1351] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.391 UTC [1351] LOG: entering standby mode logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.416 UTC [1351] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.737 UTC [1351] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.770 UTC [1351] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.772 UTC [1346] LOG: startup process (PID 1351) exited with exit code 1 logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.772 UTC [1346] LOG: aborting startup due to startup process failure logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.772 UTC [1346] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.780 UTC [1346] LOG: database system is shut down logger.go:42: 19:33:49 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:33:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:34:22 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 10m logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 10m logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 11m logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 7m32s logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 7m53s logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 6m4s logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 6m4s logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 6m18s logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 10m logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 10m logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 10m logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 9m25s logger.go:42: 19:34:23 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 12m logger.go:42: 19:34:24 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.901 UTC [1385] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.912 UTC [1385] LOG: database system is shut down logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.413 UTC [1461] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.413 UTC [1461] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.413 UTC [1461] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.438 UTC [1461] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.446 UTC [1466] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.476 UTC [1466] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.505 UTC [1466] LOG: entering standby mode logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.534 UTC [1466] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.856 UTC [1466] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.891 UTC [1466] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.892 UTC [1461] LOG: startup process (PID 1466) exited with exit code 1 logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.892 UTC [1461] LOG: aborting startup due to startup process failure logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.892 UTC [1461] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.904 UTC [1461] LOG: database system is shut down logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.415 UTC [1506] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.415 UTC [1506] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.415 UTC [1506] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.441 UTC [1506] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.449 UTC [1511] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.480 UTC [1511] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.508 UTC [1511] LOG: entering standby mode logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.537 UTC [1511] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.859 UTC [1511] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.893 UTC [1511] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.895 UTC [1506] LOG: startup process (PID 1511) exited with exit code 1 logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.895 UTC [1506] LOG: aborting startup due to startup process failure logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.895 UTC [1506] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:25 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.906 UTC [1506] LOG: database system is shut down logger.go:42: 19:34:26 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.772 UTC [1346] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:47.780 UTC [1346] LOG: database system is shut down logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.293 UTC [1422] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.294 UTC [1422] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.294 UTC [1422] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.320 UTC [1422] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.328 UTC [1427] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.355 UTC [1427] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.381 UTC [1427] LOG: entering standby mode logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.409 UTC [1427] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.729 UTC [1427] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.765 UTC [1427] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.766 UTC [1422] LOG: startup process (PID 1427) exited with exit code 1 logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.766 UTC [1422] LOG: aborting startup due to startup process failure logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.766 UTC [1422] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:07.774 UTC [1422] LOG: database system is shut down logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.293 UTC [1466] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.293 UTC [1466] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.293 UTC [1466] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.319 UTC [1466] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.327 UTC [1471] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.354 UTC [1471] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.379 UTC [1471] LOG: entering standby mode logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.405 UTC [1471] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.526 UTC [1471] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.562 UTC [1471] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.564 UTC [1466] LOG: startup process (PID 1471) exited with exit code 1 logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.564 UTC [1466] LOG: aborting startup due to startup process failure logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.564 UTC [1466] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.572 UTC [1466] LOG: database system is shut down logger.go:42: 19:34:28 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:34:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:35:00 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 11m logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 11m logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 11m logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 8m10s logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 8m31s logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 6m42s logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 6m42s logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 6m56s logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 11m logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 11m logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 11m logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 10m logger.go:42: 19:35:01 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 13m logger.go:42: 19:35:02 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.895 UTC [1506] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.906 UTC [1506] LOG: database system is shut down logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.405 UTC [1584] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.405 UTC [1584] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.405 UTC [1584] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.431 UTC [1584] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.439 UTC [1589] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.471 UTC [1589] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.500 UTC [1589] LOG: entering standby mode logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.528 UTC [1589] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.851 UTC [1589] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.887 UTC [1589] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.889 UTC [1584] LOG: startup process (PID 1589) exited with exit code 1 logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.889 UTC [1584] LOG: aborting startup due to startup process failure logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.889 UTC [1584] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.900 UTC [1584] LOG: database system is shut down logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.417 UTC [1653] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.417 UTC [1653] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.417 UTC [1653] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.443 UTC [1653] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.451 UTC [1658] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.481 UTC [1658] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.510 UTC [1658] LOG: entering standby mode logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.539 UTC [1658] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.862 UTC [1658] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.897 UTC [1658] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.899 UTC [1653] LOG: startup process (PID 1658) exited with exit code 1 logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.899 UTC [1653] LOG: aborting startup due to startup process failure logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.899 UTC [1653] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.910 UTC [1653] LOG: database system is shut down logger.go:42: 19:35:04 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.564 UTC [1466] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:17.572 UTC [1466] LOG: database system is shut down logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.310 UTC [1544] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.311 UTC [1544] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.311 UTC [1544] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.337 UTC [1544] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.345 UTC [1549] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.374 UTC [1549] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.399 UTC [1549] LOG: entering standby mode logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.429 UTC [1549] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.751 UTC [1549] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.786 UTC [1549] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.787 UTC [1544] LOG: startup process (PID 1549) exited with exit code 1 logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.787 UTC [1544] LOG: aborting startup due to startup process failure logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.787 UTC [1544] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:37.795 UTC [1544] LOG: database system is shut down logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.289 UTC [1622] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.290 UTC [1622] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.290 UTC [1622] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.316 UTC [1622] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.324 UTC [1627] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.351 UTC [1627] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.377 UTC [1627] LOG: entering standby mode logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.405 UTC [1627] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.728 UTC [1627] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.761 UTC [1627] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.763 UTC [1622] LOG: startup process (PID 1627) exited with exit code 1 logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.763 UTC [1622] LOG: aborting startup due to startup process failure logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.763 UTC [1622] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.771 UTC [1622] LOG: database system is shut down logger.go:42: 19:35:06 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:35:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:35:38 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 11m logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 11m logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 12m logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 8m49s logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 9m10s logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 7m21s logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 7m21s logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 7m35s logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 11m logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 11m logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 11m logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 10m logger.go:42: 19:35:39 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 14m logger.go:42: 19:35:41 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.899 UTC [1653] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.910 UTC [1653] LOG: database system is shut down logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.405 UTC [1730] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.406 UTC [1730] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.406 UTC [1730] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.432 UTC [1730] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.440 UTC [1735] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.472 UTC [1735] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.501 UTC [1735] LOG: entering standby mode logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.531 UTC [1735] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.857 UTC [1735] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.891 UTC [1735] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.893 UTC [1730] LOG: startup process (PID 1735) exited with exit code 1 logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.893 UTC [1730] LOG: aborting startup due to startup process failure logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.893 UTC [1730] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.905 UTC [1730] LOG: database system is shut down logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.410 UTC [1799] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.411 UTC [1799] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.411 UTC [1799] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.437 UTC [1799] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.445 UTC [1804] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.475 UTC [1804] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.504 UTC [1804] LOG: entering standby mode logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.533 UTC [1804] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.855 UTC [1804] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.889 UTC [1804] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.891 UTC [1799] LOG: startup process (PID 1804) exited with exit code 1 logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.891 UTC [1799] LOG: aborting startup due to startup process failure logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.891 UTC [1799] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.902 UTC [1799] LOG: database system is shut down logger.go:42: 19:35:42 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.763 UTC [1622] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:34:57.771 UTC [1622] LOG: database system is shut down logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.294 UTC [1699] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.294 UTC [1699] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.294 UTC [1699] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.320 UTC [1699] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.329 UTC [1704] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.356 UTC [1704] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.382 UTC [1704] LOG: entering standby mode logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.408 UTC [1704] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.729 UTC [1704] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.764 UTC [1704] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.765 UTC [1699] LOG: startup process (PID 1704) exited with exit code 1 logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.765 UTC [1699] LOG: aborting startup due to startup process failure logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.765 UTC [1699] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:17.773 UTC [1699] LOG: database system is shut down logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.301 UTC [1768] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.301 UTC [1768] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.301 UTC [1768] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.327 UTC [1768] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.335 UTC [1773] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.362 UTC [1773] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.389 UTC [1773] LOG: entering standby mode logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.415 UTC [1773] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.736 UTC [1773] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.769 UTC [1773] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.771 UTC [1768] LOG: startup process (PID 1773) exited with exit code 1 logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.771 UTC [1768] LOG: aborting startup due to startup process failure logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.771 UTC [1768] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.779 UTC [1768] LOG: database system is shut down logger.go:42: 19:35:44 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:35:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:36:17 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 12m logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 12m logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 13m logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 9m27s logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 9m48s logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 7m59s logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 7m59s logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 8m13s logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 12m logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 12m logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 12m logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 11m logger.go:42: 19:36:18 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 14m logger.go:42: 19:36:19 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.891 UTC [1799] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.902 UTC [1799] LOG: database system is shut down logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.402 UTC [1877] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.402 UTC [1877] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.402 UTC [1877] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.428 UTC [1877] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.436 UTC [1882] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.467 UTC [1882] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.496 UTC [1882] LOG: entering standby mode logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.526 UTC [1882] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.849 UTC [1882] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.885 UTC [1882] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.886 UTC [1877] LOG: startup process (PID 1882) exited with exit code 1 logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.886 UTC [1877] LOG: aborting startup due to startup process failure logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.886 UTC [1877] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.898 UTC [1877] LOG: database system is shut down logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.416 UTC [1921] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.417 UTC [1921] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.417 UTC [1921] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.443 UTC [1921] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.451 UTC [1926] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.482 UTC [1926] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.511 UTC [1926] LOG: entering standby mode logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.541 UTC [1926] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.864 UTC [1926] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.900 UTC [1926] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.901 UTC [1921] LOG: startup process (PID 1926) exited with exit code 1 logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.901 UTC [1921] LOG: aborting startup due to startup process failure logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.901 UTC [1921] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:20 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.913 UTC [1921] LOG: database system is shut down logger.go:42: 19:36:21 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.771 UTC [1768] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:37.779 UTC [1768] LOG: database system is shut down logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.294 UTC [1845] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.295 UTC [1845] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.295 UTC [1845] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.321 UTC [1845] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.329 UTC [1850] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.356 UTC [1850] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.383 UTC [1850] LOG: entering standby mode logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.409 UTC [1850] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.730 UTC [1850] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.766 UTC [1850] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.767 UTC [1845] LOG: startup process (PID 1850) exited with exit code 1 logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.767 UTC [1845] LOG: aborting startup due to startup process failure logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.767 UTC [1845] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:35:57.775 UTC [1845] LOG: database system is shut down logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.289 UTC [1889] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.289 UTC [1889] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.289 UTC [1889] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.315 UTC [1889] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.323 UTC [1894] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.349 UTC [1894] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.375 UTC [1894] LOG: entering standby mode logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.401 UTC [1894] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.725 UTC [1894] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.758 UTC [1894] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.760 UTC [1889] LOG: startup process (PID 1894) exited with exit code 1 logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.760 UTC [1889] LOG: aborting startup due to startup process failure logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.760 UTC [1889] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.767 UTC [1889] LOG: database system is shut down logger.go:42: 19:36:23 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:36:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:36:55 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 13m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 13m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 13m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 10m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 10m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 8m37s logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 8m37s logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 8m51s logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 12m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 12m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 12m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 11m logger.go:42: 19:36:56 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 15m logger.go:42: 19:36:57 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.901 UTC [1921] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:07.913 UTC [1921] LOG: database system is shut down logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.436 UTC [1999] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.437 UTC [1999] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.437 UTC [1999] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.462 UTC [1999] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.471 UTC [2004] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.503 UTC [2004] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.531 UTC [2004] LOG: entering standby mode logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.561 UTC [2004] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.884 UTC [2004] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.920 UTC [2004] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.921 UTC [1999] LOG: startup process (PID 2004) exited with exit code 1 logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.921 UTC [1999] LOG: aborting startup due to startup process failure logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.921 UTC [1999] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.933 UTC [1999] LOG: database system is shut down logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.401 UTC [2068] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.402 UTC [2068] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.402 UTC [2068] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.427 UTC [2068] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.436 UTC [2073] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.467 UTC [2073] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.497 UTC [2073] LOG: entering standby mode logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.527 UTC [2073] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.850 UTC [2073] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.885 UTC [2073] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.886 UTC [2068] LOG: startup process (PID 2073) exited with exit code 1 logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.886 UTC [2068] LOG: aborting startup due to startup process failure logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.886 UTC [2068] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:58 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.898 UTC [2068] LOG: database system is shut down logger.go:42: 19:36:59 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.771 UTC [1966] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:27.779 UTC [1966] LOG: database system is shut down logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.297 UTC [2035] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.298 UTC [2035] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.298 UTC [2035] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.324 UTC [2035] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.333 UTC [2040] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.359 UTC [2040] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.384 UTC [2040] LOG: entering standby mode logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.409 UTC [2040] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.731 UTC [2040] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.764 UTC [2040] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.766 UTC [2035] LOG: startup process (PID 2040) exited with exit code 1 logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.766 UTC [2035] LOG: aborting startup due to startup process failure logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.766 UTC [2035] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.774 UTC [2035] LOG: database system is shut down logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.291 UTC [2079] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.292 UTC [2079] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.292 UTC [2079] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.319 UTC [2079] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.327 UTC [2084] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.356 UTC [2084] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.384 UTC [2084] LOG: entering standby mode logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.410 UTC [2084] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.731 UTC [2084] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.764 UTC [2084] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.766 UTC [2079] LOG: startup process (PID 2084) exited with exit code 1 logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.766 UTC [2079] LOG: aborting startup due to startup process failure logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.766 UTC [2079] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.774 UTC [2079] LOG: database system is shut down logger.go:42: 19:37:01 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:37:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:37:33 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 13m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 13m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 14m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 10m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 11m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 9m16s logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 9m16s logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 9m30s logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 13m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 13m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 13m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 12m logger.go:42: 19:37:34 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 16m logger.go:42: 19:37:36 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.886 UTC [2068] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:47.898 UTC [2068] LOG: database system is shut down logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.417 UTC [2144] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.418 UTC [2144] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.418 UTC [2144] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.443 UTC [2144] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.452 UTC [2149] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.482 UTC [2149] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.511 UTC [2149] LOG: entering standby mode logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.541 UTC [2149] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.864 UTC [2149] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.899 UTC [2149] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.901 UTC [2144] LOG: startup process (PID 2149) exited with exit code 1 logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.901 UTC [2144] LOG: aborting startup due to startup process failure logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.901 UTC [2144] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:07.912 UTC [2144] LOG: database system is shut down logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.414 UTC [2212] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.415 UTC [2212] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.415 UTC [2212] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.440 UTC [2212] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.449 UTC [2217] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.481 UTC [2217] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.511 UTC [2217] LOG: entering standby mode logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.541 UTC [2217] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.866 UTC [2217] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.901 UTC [2217] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.903 UTC [2212] LOG: startup process (PID 2217) exited with exit code 1 logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.903 UTC [2212] LOG: aborting startup due to startup process failure logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.903 UTC [2212] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.915 UTC [2212] LOG: database system is shut down logger.go:42: 19:37:37 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.766 UTC [2079] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:36:57.774 UTC [2079] LOG: database system is shut down logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.308 UTC [2156] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.309 UTC [2156] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.309 UTC [2156] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.334 UTC [2156] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.343 UTC [2170] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.370 UTC [2170] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.396 UTC [2170] LOG: entering standby mode logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.425 UTC [2170] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.748 UTC [2170] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.782 UTC [2170] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.784 UTC [2156] LOG: startup process (PID 2170) exited with exit code 1 logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.784 UTC [2156] LOG: aborting startup due to startup process failure logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.784 UTC [2156] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:17.792 UTC [2156] LOG: database system is shut down logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.311 UTC [2225] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.311 UTC [2225] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.311 UTC [2225] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.337 UTC [2225] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.344 UTC [2239] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.370 UTC [2239] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.395 UTC [2239] LOG: entering standby mode logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.421 UTC [2239] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.743 UTC [2239] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.777 UTC [2239] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.778 UTC [2225] LOG: startup process (PID 2239) exited with exit code 1 logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.778 UTC [2225] LOG: aborting startup due to startup process failure logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.778 UTC [2225] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.786 UTC [2225] LOG: database system is shut down logger.go:42: 19:37:39 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:37:40 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:38:12 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 14m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 14m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 15m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 11m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 11m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 9m54s logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 9m54s logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 10m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 14m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 14m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 14m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 13m logger.go:42: 19:38:13 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 16m logger.go:42: 19:38:14 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.903 UTC [2212] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:27.915 UTC [2212] LOG: database system is shut down logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.413 UTC [2289] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.413 UTC [2289] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.413 UTC [2289] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.439 UTC [2289] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.448 UTC [2294] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.479 UTC [2294] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.511 UTC [2294] LOG: entering standby mode logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.540 UTC [2294] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.863 UTC [2294] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.898 UTC [2294] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.900 UTC [2289] LOG: startup process (PID 2294) exited with exit code 1 logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.900 UTC [2289] LOG: aborting startup due to startup process failure logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.900 UTC [2289] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:47.911 UTC [2289] LOG: database system is shut down logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.433 UTC [2333] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.434 UTC [2333] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.434 UTC [2333] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.460 UTC [2333] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.468 UTC [2338] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.499 UTC [2338] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.529 UTC [2338] LOG: entering standby mode logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.560 UTC [2338] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.883 UTC [2338] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.918 UTC [2338] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.920 UTC [2333] LOG: startup process (PID 2338) exited with exit code 1 logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.920 UTC [2333] LOG: aborting startup due to startup process failure logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.920 UTC [2333] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:15 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.931 UTC [2333] LOG: database system is shut down logger.go:42: 19:38:16 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.395 UTC [2239] LOG: entering standby mode logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.421 UTC [2239] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.743 UTC [2239] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.777 UTC [2239] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.778 UTC [2225] LOG: startup process (PID 2239) exited with exit code 1 logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.778 UTC [2225] LOG: aborting startup due to startup process failure logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.778 UTC [2225] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:37.786 UTC [2225] LOG: database system is shut down logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.308 UTC [2302] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.308 UTC [2302] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.308 UTC [2302] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.334 UTC [2302] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.342 UTC [2317] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.369 UTC [2317] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.396 UTC [2317] LOG: entering standby mode logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.423 UTC [2317] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.746 UTC [2317] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.780 UTC [2317] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.781 UTC [2302] LOG: startup process (PID 2317) exited with exit code 1 logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.781 UTC [2302] LOG: aborting startup due to startup process failure logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.781 UTC [2302] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:37:57.789 UTC [2302] LOG: database system is shut down logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.302 UTC [2373] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.303 UTC [2373] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.303 UTC [2373] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.328 UTC [2373] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.336 UTC [2387] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.364 UTC [2387] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.390 UTC [2387] LOG: entering standby mode logger.go:42: 19:38:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.417 UTC [2387] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:18 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:38:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:38:50 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 15m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 15m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 15m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 12m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 12m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 10m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 10m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 10m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 14m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 14m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 14m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 13m logger.go:42: 19:38:51 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 17m logger.go:42: 19:38:52 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.898 UTC [2410] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.909 UTC [2410] LOG: database system is shut down logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.400 UTC [2463] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.401 UTC [2463] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.401 UTC [2463] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.426 UTC [2463] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.434 UTC [2468] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.464 UTC [2468] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.493 UTC [2468] LOG: entering standby mode logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.522 UTC [2468] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.844 UTC [2468] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.878 UTC [2468] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.880 UTC [2463] LOG: startup process (PID 2468) exited with exit code 1 logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.880 UTC [2463] LOG: aborting startup due to startup process failure logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.880 UTC [2463] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.892 UTC [2463] LOG: database system is shut down logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.422 UTC [2506] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.423 UTC [2506] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.423 UTC [2506] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.449 UTC [2506] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.457 UTC [2511] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.490 UTC [2511] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.518 UTC [2511] LOG: entering standby mode logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.548 UTC [2511] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.870 UTC [2511] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.905 UTC [2511] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.906 UTC [2506] LOG: startup process (PID 2511) exited with exit code 1 logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.906 UTC [2506] LOG: aborting startup due to startup process failure logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.906 UTC [2506] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.918 UTC [2506] LOG: database system is shut down logger.go:42: 19:38:54 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.775 UTC [2373] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:17.783 UTC [2373] LOG: database system is shut down logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.317 UTC [2425] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.318 UTC [2425] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.318 UTC [2425] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.343 UTC [2425] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.351 UTC [2439] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.379 UTC [2439] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.407 UTC [2439] LOG: entering standby mode logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.435 UTC [2439] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.757 UTC [2439] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.791 UTC [2439] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.792 UTC [2425] LOG: startup process (PID 2439) exited with exit code 1 logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.792 UTC [2425] LOG: aborting startup due to startup process failure logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.792 UTC [2425] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:27.800 UTC [2425] LOG: database system is shut down logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.303 UTC [2478] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.304 UTC [2478] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.304 UTC [2478] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.329 UTC [2478] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.337 UTC [2483] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.364 UTC [2483] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.390 UTC [2483] LOG: entering standby mode logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.417 UTC [2483] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.738 UTC [2483] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.773 UTC [2483] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.775 UTC [2478] LOG: startup process (PID 2483) exited with exit code 1 logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.775 UTC [2478] LOG: aborting startup due to startup process failure logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.775 UTC [2478] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:37.783 UTC [2478] LOG: database system is shut down logger.go:42: 19:38:56 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:38:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:39:28 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 15m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 15m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 16m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 12m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 13m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 11m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 11m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 11m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 15m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 15m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 15m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 14m logger.go:42: 19:39:29 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 18m logger.go:42: 19:39:31 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:57.891 UTC [2583] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:57.902 UTC [2583] LOG: database system is shut down logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.435 UTC [2627] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.436 UTC [2627] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.436 UTC [2627] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.461 UTC [2627] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.470 UTC [2632] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.500 UTC [2632] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.529 UTC [2632] LOG: entering standby mode logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.558 UTC [2632] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.880 UTC [2632] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.915 UTC [2632] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.916 UTC [2627] LOG: startup process (PID 2632) exited with exit code 1 logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.916 UTC [2627] LOG: aborting startup due to startup process failure logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.916 UTC [2627] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.928 UTC [2627] LOG: database system is shut down logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.414 UTC [2696] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.415 UTC [2696] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.415 UTC [2696] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.440 UTC [2696] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.449 UTC [2701] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.480 UTC [2701] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.508 UTC [2701] LOG: entering standby mode logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.538 UTC [2701] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.860 UTC [2701] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.895 UTC [2701] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.897 UTC [2696] LOG: startup process (PID 2701) exited with exit code 1 logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.897 UTC [2696] LOG: aborting startup due to startup process failure logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.897 UTC [2696] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.908 UTC [2696] LOG: database system is shut down logger.go:42: 19:39:32 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:57.767 UTC [2554] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:38:57.775 UTC [2554] LOG: database system is shut down logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.289 UTC [2598] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.290 UTC [2598] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.290 UTC [2598] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.315 UTC [2598] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.324 UTC [2603] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.350 UTC [2603] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.375 UTC [2603] LOG: entering standby mode logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.402 UTC [2603] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.724 UTC [2603] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.757 UTC [2603] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.758 UTC [2598] LOG: startup process (PID 2603) exited with exit code 1 logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.758 UTC [2598] LOG: aborting startup due to startup process failure logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.758 UTC [2598] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:07.766 UTC [2598] LOG: database system is shut down logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.297 UTC [2667] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.298 UTC [2667] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.298 UTC [2667] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.324 UTC [2667] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.332 UTC [2672] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.361 UTC [2672] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.387 UTC [2672] LOG: entering standby mode logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.414 UTC [2672] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.737 UTC [2672] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.770 UTC [2672] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.772 UTC [2667] LOG: startup process (PID 2672) exited with exit code 1 logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.772 UTC [2667] LOG: aborting startup due to startup process failure logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.772 UTC [2667] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.781 UTC [2667] LOG: database system is shut down logger.go:42: 19:39:34 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:39:35 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:40:07 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 16m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 16m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 16m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 13m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 13m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 11m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 11m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 12m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 16m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 16m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 16m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 15m logger.go:42: 19:40:08 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 18m logger.go:42: 19:40:09 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.897 UTC [2696] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:27.908 UTC [2696] LOG: database system is shut down logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.432 UTC [2772] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.433 UTC [2772] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.433 UTC [2772] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.458 UTC [2772] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.467 UTC [2777] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.497 UTC [2777] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.526 UTC [2777] LOG: entering standby mode logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.553 UTC [2777] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.876 UTC [2777] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.910 UTC [2777] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.912 UTC [2772] LOG: startup process (PID 2777) exited with exit code 1 logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.912 UTC [2772] LOG: aborting startup due to startup process failure logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.912 UTC [2772] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.923 UTC [2772] LOG: database system is shut down logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.443 UTC [2840] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.444 UTC [2840] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.444 UTC [2840] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.470 UTC [2840] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.479 UTC [2845] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.511 UTC [2845] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.539 UTC [2845] LOG: entering standby mode logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.568 UTC [2845] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.891 UTC [2845] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.926 UTC [2845] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.927 UTC [2840] LOG: startup process (PID 2845) exited with exit code 1 logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.927 UTC [2840] LOG: aborting startup due to startup process failure logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.927 UTC [2840] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.939 UTC [2840] LOG: database system is shut down logger.go:42: 19:40:11 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:37.750 UTC [2719] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:37.759 UTC [2719] LOG: database system is shut down logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.300 UTC [2772] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.301 UTC [2772] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.301 UTC [2772] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.327 UTC [2772] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.335 UTC [2777] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.362 UTC [2777] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.389 UTC [2777] LOG: entering standby mode logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.415 UTC [2777] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.737 UTC [2777] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.771 UTC [2777] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.773 UTC [2772] LOG: startup process (PID 2777) exited with exit code 1 logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.773 UTC [2772] LOG: aborting startup due to startup process failure logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.773 UTC [2772] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:39:47.780 UTC [2772] LOG: database system is shut down logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.295 UTC [2840] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.295 UTC [2840] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.295 UTC [2840] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.321 UTC [2840] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.330 UTC [2845] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.356 UTC [2845] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.383 UTC [2845] LOG: entering standby mode logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.408 UTC [2845] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.730 UTC [2845] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.764 UTC [2845] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.766 UTC [2840] LOG: startup process (PID 2845) exited with exit code 1 logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.766 UTC [2840] LOG: aborting startup due to startup process failure logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.766 UTC [2840] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:12 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:07.773 UTC [2840] LOG: database system is shut down logger.go:42: 19:40:13 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:40:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:40:45 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 17m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 17m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 17m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 13m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 14m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 12m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 12m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 12m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 16m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 16m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 16m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 15m logger.go:42: 19:40:46 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 19m logger.go:42: 19:40:47 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:17.893 UTC [2892] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:17.905 UTC [2892] LOG: database system is shut down logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.429 UTC [2961] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.430 UTC [2961] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.430 UTC [2961] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.455 UTC [2961] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.464 UTC [2966] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.495 UTC [2966] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.522 UTC [2966] LOG: entering standby mode logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.553 UTC [2966] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.877 UTC [2966] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.912 UTC [2966] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.913 UTC [2961] LOG: startup process (PID 2966) exited with exit code 1 logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.913 UTC [2961] LOG: aborting startup due to startup process failure logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.913 UTC [2961] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.925 UTC [2961] LOG: database system is shut down logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.426 UTC [3005] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.427 UTC [3005] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.427 UTC [3005] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.453 UTC [3005] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.462 UTC [3010] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.493 UTC [3010] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.522 UTC [3010] LOG: entering standby mode logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.551 UTC [3010] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.874 UTC [3010] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.910 UTC [3010] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.911 UTC [3005] LOG: startup process (PID 3010) exited with exit code 1 logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.911 UTC [3005] LOG: aborting startup due to startup process failure logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.911 UTC [3005] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.923 UTC [3005] LOG: database system is shut down logger.go:42: 19:40:49 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:17.759 UTC [2892] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:17.766 UTC [2892] LOG: database system is shut down logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.288 UTC [2960] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.289 UTC [2960] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.289 UTC [2960] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.314 UTC [2960] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.321 UTC [2965] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.348 UTC [2965] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.374 UTC [2965] LOG: entering standby mode logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.401 UTC [2965] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.722 UTC [2965] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.755 UTC [2965] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.757 UTC [2960] LOG: startup process (PID 2965) exited with exit code 1 logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.757 UTC [2960] LOG: aborting startup due to startup process failure logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.757 UTC [2960] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.765 UTC [2960] LOG: database system is shut down logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.292 UTC [3004] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.293 UTC [3004] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.293 UTC [3004] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.319 UTC [3004] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.327 UTC [3009] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.354 UTC [3009] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.380 UTC [3009] LOG: entering standby mode logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.405 UTC [3009] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.726 UTC [3009] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.759 UTC [3009] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.760 UTC [3004] LOG: startup process (PID 3009) exited with exit code 1 logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.760 UTC [3004] LOG: aborting startup due to startup process failure logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.760 UTC [3004] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.768 UTC [3004] LOG: database system is shut down logger.go:42: 19:40:51 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:40:52 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:41:23 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 17m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 17m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 18m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 14m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 14m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 13m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 13m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 13m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 17m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 17m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 17m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 16m logger.go:42: 19:41:24 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 19m logger.go:42: 19:41:26 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.913 UTC [2961] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:37.925 UTC [2961] LOG: database system is shut down logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.426 UTC [3005] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.427 UTC [3005] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.427 UTC [3005] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.453 UTC [3005] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.462 UTC [3010] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.493 UTC [3010] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.522 UTC [3010] LOG: entering standby mode logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.551 UTC [3010] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.874 UTC [3010] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.910 UTC [3010] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.911 UTC [3005] LOG: startup process (PID 3010) exited with exit code 1 logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.911 UTC [3005] LOG: aborting startup due to startup process failure logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.911 UTC [3005] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:47.923 UTC [3005] LOG: database system is shut down logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.434 UTC [3082] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.435 UTC [3082] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.435 UTC [3082] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.461 UTC [3082] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.469 UTC [3087] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.501 UTC [3087] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.528 UTC [3087] LOG: entering standby mode logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.556 UTC [3087] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.879 UTC [3087] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.914 UTC [3087] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.916 UTC [3082] LOG: startup process (PID 3087) exited with exit code 1 logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.916 UTC [3082] LOG: aborting startup due to startup process failure logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.916 UTC [3082] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:27 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:07.927 UTC [3082] LOG: database system is shut down logger.go:42: 19:41:28 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:57.764 UTC [3056] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:40:57.771 UTC [3056] LOG: database system is shut down logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.286 UTC [3125] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.286 UTC [3125] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.286 UTC [3125] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.315 UTC [3125] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.323 UTC [3130] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.350 UTC [3130] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.376 UTC [3130] LOG: entering standby mode logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.401 UTC [3130] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.722 UTC [3130] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.755 UTC [3130] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.756 UTC [3125] LOG: startup process (PID 3130) exited with exit code 1 logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.756 UTC [3125] LOG: aborting startup due to startup process failure logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.756 UTC [3125] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:17.764 UTC [3125] LOG: database system is shut down logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.291 UTC [3169] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.291 UTC [3169] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.291 UTC [3169] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.317 UTC [3169] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.325 UTC [3174] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.351 UTC [3174] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.377 UTC [3174] LOG: entering standby mode logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.404 UTC [3174] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.726 UTC [3174] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.760 UTC [3174] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.762 UTC [3169] LOG: startup process (PID 3174) exited with exit code 1 logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.762 UTC [3169] LOG: aborting startup due to startup process failure logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.762 UTC [3169] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.770 UTC [3169] LOG: database system is shut down logger.go:42: 19:41:29 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:41:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:42:02 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 18m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 18m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 18m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 15m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 15m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 13m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 13m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 13m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 18m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 18m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 18m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 17m logger.go:42: 19:42:03 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 20m logger.go:42: 19:42:04 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.917 UTC [3152] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.929 UTC [3152] LOG: database system is shut down logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.434 UTC [3230] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.435 UTC [3230] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.435 UTC [3230] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.460 UTC [3230] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.470 UTC [3235] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.501 UTC [3235] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.530 UTC [3235] LOG: entering standby mode logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.560 UTC [3235] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.891 UTC [3235] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.927 UTC [3235] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.928 UTC [3230] LOG: startup process (PID 3235) exited with exit code 1 logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.928 UTC [3230] LOG: aborting startup due to startup process failure logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.929 UTC [3230] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.940 UTC [3230] LOG: database system is shut down logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.447 UTC [3274] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.447 UTC [3274] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.447 UTC [3274] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.473 UTC [3274] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.483 UTC [3279] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.514 UTC [3279] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.542 UTC [3279] LOG: entering standby mode logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.573 UTC [3279] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.896 UTC [3279] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.933 UTC [3279] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.935 UTC [3274] LOG: startup process (PID 3279) exited with exit code 1 logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.935 UTC [3274] LOG: aborting startup due to startup process failure logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.935 UTC [3274] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.946 UTC [3274] LOG: database system is shut down logger.go:42: 19:42:06 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.762 UTC [3169] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:27.770 UTC [3169] LOG: database system is shut down logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.295 UTC [3246] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.296 UTC [3246] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.296 UTC [3246] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.322 UTC [3246] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.330 UTC [3251] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.357 UTC [3251] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.382 UTC [3251] LOG: entering standby mode logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.411 UTC [3251] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.733 UTC [3251] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.766 UTC [3251] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.768 UTC [3246] LOG: startup process (PID 3251) exited with exit code 1 logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.768 UTC [3246] LOG: aborting startup due to startup process failure logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.768 UTC [3246] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:47.776 UTC [3246] LOG: database system is shut down logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.294 UTC [3290] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.295 UTC [3290] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.295 UTC [3290] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.321 UTC [3290] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.329 UTC [3295] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.354 UTC [3295] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.377 UTC [3295] LOG: entering standby mode logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.402 UTC [3295] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.723 UTC [3295] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.757 UTC [3295] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.758 UTC [3290] LOG: startup process (PID 3295) exited with exit code 1 logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.758 UTC [3290] LOG: aborting startup due to startup process failure logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.758 UTC [3290] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:07 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.766 UTC [3290] LOG: database system is shut down logger.go:42: 19:42:08 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:42:09 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:42:40 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 19m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 19m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 19m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 15m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 16m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 14m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 14m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 14m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 18m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 18m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 18m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 17m logger.go:42: 19:42:41 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 21m logger.go:42: 19:42:42 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.935 UTC [3274] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.946 UTC [3274] LOG: database system is shut down logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.441 UTC [3352] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.442 UTC [3352] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.442 UTC [3352] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.469 UTC [3352] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.480 UTC [3357] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.513 UTC [3357] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.542 UTC [3357] LOG: entering standby mode logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.572 UTC [3357] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.895 UTC [3357] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.932 UTC [3357] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.934 UTC [3352] LOG: startup process (PID 3357) exited with exit code 1 logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.934 UTC [3352] LOG: aborting startup due to startup process failure logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.934 UTC [3352] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.946 UTC [3352] LOG: database system is shut down logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.415 UTC [3421] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.415 UTC [3421] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.415 UTC [3421] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.441 UTC [3421] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.449 UTC [3426] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.481 UTC [3426] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.509 UTC [3426] LOG: entering standby mode logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.538 UTC [3426] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.861 UTC [3426] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.899 UTC [3426] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.901 UTC [3421] LOG: startup process (PID 3426) exited with exit code 1 logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.901 UTC [3421] LOG: aborting startup due to startup process failure logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.901 UTC [3421] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.912 UTC [3421] LOG: database system is shut down logger.go:42: 19:42:44 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.758 UTC [3290] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:41:57.766 UTC [3290] LOG: database system is shut down logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.313 UTC [3367] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.314 UTC [3367] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.314 UTC [3367] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.340 UTC [3367] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.348 UTC [3381] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.375 UTC [3381] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.398 UTC [3381] LOG: entering standby mode logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.426 UTC [3381] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.747 UTC [3381] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.780 UTC [3381] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.782 UTC [3367] LOG: startup process (PID 3381) exited with exit code 1 logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.782 UTC [3367] LOG: aborting startup due to startup process failure logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.782 UTC [3367] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:17.790 UTC [3367] LOG: database system is shut down logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.295 UTC [3437] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.296 UTC [3437] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.296 UTC [3437] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.321 UTC [3437] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.330 UTC [3451] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.356 UTC [3451] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.382 UTC [3451] LOG: entering standby mode logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.408 UTC [3451] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.729 UTC [3451] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.762 UTC [3451] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.763 UTC [3437] LOG: startup process (PID 3451) exited with exit code 1 logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.763 UTC [3437] LOG: aborting startup due to startup process failure logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.763 UTC [3437] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:37.772 UTC [3437] LOG: database system is shut down logger.go:42: 19:42:46 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:42:47 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 19m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 19m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 20m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 16m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 16m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 15m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 15m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 15m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 19m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 19m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 19m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 18m logger.go:42: 19:43:19 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 21m logger.go:42: 19:43:21 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:47.994 UTC [3473] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:48.006 UTC [3473] LOG: database system is shut down logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.488 UTC [3517] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.489 UTC [3517] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.489 UTC [3517] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.514 UTC [3517] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.523 UTC [3522] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.555 UTC [3522] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.585 UTC [3522] LOG: entering standby mode logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.614 UTC [3522] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.939 UTC [3522] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.977 UTC [3522] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.978 UTC [3517] LOG: startup process (PID 3522) exited with exit code 1 logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.978 UTC [3517] LOG: aborting startup due to startup process failure logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.978 UTC [3517] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.990 UTC [3517] LOG: database system is shut down logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.581 UTC [3585] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.582 UTC [3585] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.582 UTC [3585] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.610 UTC [3585] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.621 UTC [3590] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.674 UTC [3590] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.712 UTC [3590] LOG: entering standby mode logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.748 UTC [3590] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:18.087 UTC [3590] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:18.124 UTC [3590] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:18.126 UTC [3585] LOG: startup process (PID 3590) exited with exit code 1 logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:18.126 UTC [3585] LOG: aborting startup due to startup process failure logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:18.126 UTC [3585] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:22 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:18.137 UTC [3585] LOG: database system is shut down logger.go:42: 19:43:23 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:47.780 UTC [3489] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:47.788 UTC [3489] LOG: database system is shut down logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.355 UTC [3533] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.356 UTC [3533] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.356 UTC [3533] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.382 UTC [3533] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.390 UTC [3547] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.417 UTC [3547] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.444 UTC [3547] LOG: entering standby mode logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.469 UTC [3547] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.792 UTC [3547] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.826 UTC [3547] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.828 UTC [3533] LOG: startup process (PID 3547) exited with exit code 1 logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.828 UTC [3533] LOG: aborting startup due to startup process failure logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.828 UTC [3533] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:42:57.836 UTC [3533] LOG: database system is shut down logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.305 UTC [3602] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.305 UTC [3602] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.305 UTC [3602] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.331 UTC [3602] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.339 UTC [3616] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.367 UTC [3616] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.393 UTC [3616] LOG: entering standby mode logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.420 UTC [3616] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.741 UTC [3616] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.775 UTC [3616] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.776 UTC [3602] LOG: startup process (PID 3616) exited with exit code 1 logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.776 UTC [3602] LOG: aborting startup due to startup process failure logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.776 UTC [3602] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:17.784 UTC [3602] LOG: database system is shut down logger.go:42: 19:43:24 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:43:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:43:57 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 20m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 20m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 20m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 17m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 17m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 15m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 15m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 15m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 19m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 19m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 19m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 19m logger.go:42: 19:43:58 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 22m logger.go:42: 19:43:59 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:27.969 UTC [3637] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:27.981 UTC [3637] LOG: database system is shut down logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.430 UTC [3680] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.430 UTC [3680] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.430 UTC [3680] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.456 UTC [3680] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.466 UTC [3685] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.498 UTC [3685] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.527 UTC [3685] LOG: entering standby mode logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.558 UTC [3685] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.882 UTC [3685] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.918 UTC [3685] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.920 UTC [3680] LOG: startup process (PID 3685) exited with exit code 1 logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.920 UTC [3680] LOG: aborting startup due to startup process failure logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.920 UTC [3680] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.932 UTC [3680] LOG: database system is shut down logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.499 UTC [3746] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.500 UTC [3746] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.500 UTC [3746] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.537 UTC [3746] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.545 UTC [3751] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.579 UTC [3751] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.608 UTC [3751] LOG: entering standby mode logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.642 UTC [3751] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.972 UTC [3751] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:58.008 UTC [3751] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:58.010 UTC [3746] LOG: startup process (PID 3751) exited with exit code 1 logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:58.010 UTC [3746] LOG: aborting startup due to startup process failure logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:58.010 UTC [3746] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:00 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:58.022 UTC [3746] LOG: database system is shut down logger.go:42: 19:44:01 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:27.801 UTC [3654] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:27.809 UTC [3654] LOG: database system is shut down logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.294 UTC [3707] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.294 UTC [3707] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.294 UTC [3707] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.320 UTC [3707] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.328 UTC [3712] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.354 UTC [3712] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.379 UTC [3712] LOG: entering standby mode logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.405 UTC [3712] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.726 UTC [3712] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.760 UTC [3712] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.762 UTC [3707] LOG: startup process (PID 3712) exited with exit code 1 logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.762 UTC [3707] LOG: aborting startup due to startup process failure logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.762 UTC [3707] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:37.770 UTC [3707] LOG: database system is shut down logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.305 UTC [3775] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.306 UTC [3775] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.306 UTC [3775] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.331 UTC [3775] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.339 UTC [3780] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.365 UTC [3780] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.389 UTC [3780] LOG: entering standby mode logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.417 UTC [3780] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.739 UTC [3780] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.771 UTC [3780] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.773 UTC [3775] LOG: startup process (PID 3780) exited with exit code 1 logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.773 UTC [3775] LOG: aborting startup due to startup process failure logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.773 UTC [3775] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:02 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.780 UTC [3775] LOG: database system is shut down logger.go:42: 19:44:03 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:44:04 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:44:35 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 20m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 20m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 21m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 17m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 18m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 16m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 16m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 16m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 20m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 20m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 20m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 19m logger.go:42: 19:44:36 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 23m logger.go:42: 19:44:38 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:58.010 UTC [3746] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:58.022 UTC [3746] LOG: database system is shut down logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.578 UTC [3823] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.579 UTC [3823] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.579 UTC [3823] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.609 UTC [3823] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.621 UTC [3828] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.659 UTC [3828] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.695 UTC [3828] LOG: entering standby mode logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.727 UTC [3828] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:18.071 UTC [3828] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:18.110 UTC [3828] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:18.113 UTC [3823] LOG: startup process (PID 3828) exited with exit code 1 logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:18.113 UTC [3823] LOG: aborting startup due to startup process failure logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:18.113 UTC [3823] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:18.125 UTC [3823] LOG: database system is shut down logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.457 UTC [3891] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.458 UTC [3891] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.458 UTC [3891] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.484 UTC [3891] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.493 UTC [3896] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.523 UTC [3896] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.551 UTC [3896] LOG: entering standby mode logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.584 UTC [3896] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.910 UTC [3896] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.946 UTC [3896] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.948 UTC [3891] LOG: startup process (PID 3896) exited with exit code 1 logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.948 UTC [3891] LOG: aborting startup due to startup process failure logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.948 UTC [3891] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.959 UTC [3891] LOG: database system is shut down logger.go:42: 19:44:39 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.773 UTC [3775] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:43:57.780 UTC [3775] LOG: database system is shut down logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.289 UTC [3853] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.290 UTC [3853] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.290 UTC [3853] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.316 UTC [3853] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.323 UTC [3858] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.349 UTC [3858] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.376 UTC [3858] LOG: entering standby mode logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.402 UTC [3858] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.723 UTC [3858] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.758 UTC [3858] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.760 UTC [3853] LOG: startup process (PID 3858) exited with exit code 1 logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.760 UTC [3853] LOG: aborting startup due to startup process failure logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.760 UTC [3853] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:17.767 UTC [3853] LOG: database system is shut down logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.356 UTC [3923] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.356 UTC [3923] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.356 UTC [3923] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.382 UTC [3923] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.391 UTC [3928] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.419 UTC [3928] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.445 UTC [3928] LOG: entering standby mode logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.471 UTC [3928] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.793 UTC [3928] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.827 UTC [3928] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.829 UTC [3923] LOG: startup process (PID 3928) exited with exit code 1 logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.829 UTC [3923] LOG: aborting startup due to startup process failure logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.829 UTC [3923] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.838 UTC [3923] LOG: database system is shut down logger.go:42: 19:44:41 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:44:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:45:14 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 21m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 21m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 22m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 18m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 18m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 16m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 16m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 17m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 21m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 21m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 21m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 20m logger.go:42: 19:45:15 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 23m logger.go:42: 19:45:16 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.948 UTC [3891] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.959 UTC [3891] LOG: database system is shut down logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.406 UTC [3968] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.407 UTC [3968] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.407 UTC [3968] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.435 UTC [3968] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.443 UTC [3973] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.474 UTC [3973] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.505 UTC [3973] LOG: entering standby mode logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.537 UTC [3973] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.859 UTC [3973] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.893 UTC [3973] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.895 UTC [3968] LOG: startup process (PID 3973) exited with exit code 1 logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.895 UTC [3968] LOG: aborting startup due to startup process failure logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.895 UTC [3968] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.906 UTC [3968] LOG: database system is shut down logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.439 UTC [4021] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.439 UTC [4021] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.439 UTC [4021] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.465 UTC [4021] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.474 UTC [4026] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.508 UTC [4026] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.539 UTC [4026] LOG: entering standby mode logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.571 UTC [4026] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.896 UTC [4026] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.932 UTC [4026] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.934 UTC [4021] LOG: startup process (PID 4026) exited with exit code 1 logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.934 UTC [4021] LOG: aborting startup due to startup process failure logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.934 UTC [4021] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:17 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.946 UTC [4021] LOG: database system is shut down logger.go:42: 19:45:18 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.829 UTC [3923] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:37.838 UTC [3923] LOG: database system is shut down logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.322 UTC [4009] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.323 UTC [4009] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.323 UTC [4009] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.349 UTC [4009] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.358 UTC [4014] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.387 UTC [4014] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.414 UTC [4014] LOG: entering standby mode logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.442 UTC [4014] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.765 UTC [4014] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.800 UTC [4014] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.802 UTC [4009] LOG: startup process (PID 4014) exited with exit code 1 logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.802 UTC [4009] LOG: aborting startup due to startup process failure logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.802 UTC [4009] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:44:57.810 UTC [4009] LOG: database system is shut down logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.330 UTC [4052] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.330 UTC [4052] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.330 UTC [4052] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.358 UTC [4052] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.366 UTC [4057] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.395 UTC [4057] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.423 UTC [4057] LOG: entering standby mode logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.450 UTC [4057] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.773 UTC [4057] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.810 UTC [4057] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.812 UTC [4052] LOG: startup process (PID 4057) exited with exit code 1 logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.812 UTC [4052] LOG: aborting startup due to startup process failure logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.812 UTC [4052] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.821 UTC [4052] LOG: database system is shut down logger.go:42: 19:45:20 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:45:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:45:52 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 22m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 22m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 22m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 19m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 19m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 17m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 17m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 17m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 21m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 21m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 21m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 20m logger.go:42: 19:45:53 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 24m logger.go:42: 19:45:54 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.934 UTC [4021] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:07.946 UTC [4021] LOG: database system is shut down logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.487 UTC [4098] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.487 UTC [4098] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.488 UTC [4098] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.513 UTC [4098] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.522 UTC [4103] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.554 UTC [4103] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.584 UTC [4103] LOG: entering standby mode logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.615 UTC [4103] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.939 UTC [4103] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.974 UTC [4103] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.976 UTC [4098] LOG: startup process (PID 4103) exited with exit code 1 logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.977 UTC [4098] LOG: aborting startup due to startup process failure logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.977 UTC [4098] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.990 UTC [4098] LOG: database system is shut down logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.432 UTC [4167] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.433 UTC [4167] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.433 UTC [4167] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.459 UTC [4167] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.469 UTC [4172] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.505 UTC [4172] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.534 UTC [4172] LOG: entering standby mode logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.566 UTC [4172] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.890 UTC [4172] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.926 UTC [4172] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.928 UTC [4167] LOG: startup process (PID 4172) exited with exit code 1 logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.928 UTC [4167] LOG: aborting startup due to startup process failure logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.928 UTC [4167] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.941 UTC [4167] LOG: database system is shut down logger.go:42: 19:45:56 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.794 UTC [4129] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:27.802 UTC [4129] LOG: database system is shut down logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.307 UTC [4207] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.307 UTC [4207] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.307 UTC [4207] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.334 UTC [4207] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.343 UTC [4212] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.373 UTC [4212] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.399 UTC [4212] LOG: entering standby mode logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.428 UTC [4212] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.750 UTC [4212] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.785 UTC [4212] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.787 UTC [4207] LOG: startup process (PID 4212) exited with exit code 1 logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.787 UTC [4207] LOG: aborting startup due to startup process failure logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.787 UTC [4207] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:47.795 UTC [4207] LOG: database system is shut down logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.312 UTC [4251] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.312 UTC [4251] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.312 UTC [4251] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.338 UTC [4251] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.347 UTC [4256] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.374 UTC [4256] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.400 UTC [4256] LOG: entering standby mode logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.428 UTC [4256] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.751 UTC [4256] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.787 UTC [4256] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.788 UTC [4251] LOG: startup process (PID 4256) exited with exit code 1 logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.788 UTC [4251] LOG: aborting startup due to startup process failure logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.788 UTC [4251] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:57 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.796 UTC [4251] LOG: database system is shut down logger.go:42: 19:45:58 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:45:59 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 22m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 22m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 23m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 19m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 20m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 18m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 18m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 18m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 22m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 22m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 22m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 21m logger.go:42: 19:46:31 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 25m logger.go:42: 19:46:33 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.969 UTC [4219] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.982 UTC [4219] LOG: database system is shut down logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.468 UTC [4297] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.469 UTC [4297] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.469 UTC [4297] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.494 UTC [4297] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.503 UTC [4302] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.536 UTC [4302] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.566 UTC [4302] LOG: entering standby mode logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.600 UTC [4302] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.936 UTC [4302] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.971 UTC [4302] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.973 UTC [4297] LOG: startup process (PID 4302) exited with exit code 1 logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.973 UTC [4297] LOG: aborting startup due to startup process failure logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.973 UTC [4297] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:17.985 UTC [4297] LOG: database system is shut down logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.423 UTC [4340] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.424 UTC [4340] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.424 UTC [4340] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.450 UTC [4340] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.458 UTC [4345] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.490 UTC [4345] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.518 UTC [4345] LOG: entering standby mode logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.547 UTC [4345] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.870 UTC [4345] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.905 UTC [4345] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.907 UTC [4340] LOG: startup process (PID 4345) exited with exit code 1 logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.907 UTC [4340] LOG: aborting startup due to startup process failure logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.907 UTC [4340] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:34 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.918 UTC [4340] LOG: database system is shut down logger.go:42: 19:46:35 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.788 UTC [4251] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:45:57.796 UTC [4251] LOG: database system is shut down logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.311 UTC [4303] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.312 UTC [4303] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.312 UTC [4303] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.337 UTC [4303] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.346 UTC [4308] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.373 UTC [4308] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.398 UTC [4308] LOG: entering standby mode logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.425 UTC [4308] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.747 UTC [4308] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.780 UTC [4308] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.782 UTC [4303] LOG: startup process (PID 4308) exited with exit code 1 logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.782 UTC [4303] LOG: aborting startup due to startup process failure logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.782 UTC [4303] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:07.790 UTC [4303] LOG: database system is shut down logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.296 UTC [4372] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.297 UTC [4372] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.297 UTC [4372] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.324 UTC [4372] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.337 UTC [4377] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.388 UTC [4377] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.415 UTC [4377] LOG: entering standby mode logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.443 UTC [4377] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.764 UTC [4377] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.799 UTC [4377] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.800 UTC [4372] LOG: startup process (PID 4377) exited with exit code 1 logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.800 UTC [4372] LOG: aborting startup due to startup process failure logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.800 UTC [4372] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:36 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.808 UTC [4372] LOG: database system is shut down logger.go:42: 19:46:37 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:46:38 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:47:09 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 23m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 23m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 23m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 20m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 20m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 18m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 18m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 19m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 23m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 23m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 23m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 22m logger.go:42: 19:47:10 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 25m logger.go:42: 19:47:11 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.907 UTC [4340] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:27.918 UTC [4340] LOG: database system is shut down logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.445 UTC [4417] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.446 UTC [4417] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.446 UTC [4417] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.471 UTC [4417] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.480 UTC [4422] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.514 UTC [4422] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.544 UTC [4422] LOG: entering standby mode logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.576 UTC [4422] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.908 UTC [4422] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.943 UTC [4422] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.945 UTC [4417] LOG: startup process (PID 4422) exited with exit code 1 logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.945 UTC [4417] LOG: aborting startup due to startup process failure logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.945 UTC [4417] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:47.957 UTC [4417] LOG: database system is shut down logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.493 UTC [4486] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.494 UTC [4486] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.494 UTC [4486] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.520 UTC [4486] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.530 UTC [4491] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.564 UTC [4491] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.594 UTC [4491] LOG: entering standby mode logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.626 UTC [4491] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.956 UTC [4491] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.991 UTC [4491] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.993 UTC [4486] LOG: startup process (PID 4491) exited with exit code 1 logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.993 UTC [4486] LOG: aborting startup due to startup process failure logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.993 UTC [4486] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:08.006 UTC [4486] LOG: database system is shut down logger.go:42: 19:47:13 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:37.765 UTC [4424] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:37.773 UTC [4424] LOG: database system is shut down logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.304 UTC [4493] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.304 UTC [4493] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.304 UTC [4493] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.330 UTC [4493] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.339 UTC [4498] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.368 UTC [4498] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.394 UTC [4498] LOG: entering standby mode logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.420 UTC [4498] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.743 UTC [4498] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.777 UTC [4498] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.778 UTC [4493] LOG: startup process (PID 4498) exited with exit code 1 logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.778 UTC [4493] LOG: aborting startup due to startup process failure logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.778 UTC [4493] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:46:57.786 UTC [4493] LOG: database system is shut down logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.310 UTC [4537] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.310 UTC [4537] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.311 UTC [4537] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.336 UTC [4537] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.345 UTC [4551] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.372 UTC [4551] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.400 UTC [4551] LOG: entering standby mode logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.427 UTC [4551] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.749 UTC [4551] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.784 UTC [4551] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.786 UTC [4537] LOG: startup process (PID 4551) exited with exit code 1 logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.786 UTC [4537] LOG: aborting startup due to startup process failure logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.786 UTC [4537] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:14 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:07.794 UTC [4537] LOG: database system is shut down logger.go:42: 19:47:15 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:47:16 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 24m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 24m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 24m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 20m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 21m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 19m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 19m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 19m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 23m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 23m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 23m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 22m logger.go:42: 19:47:48 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 26m logger.go:42: 19:47:50 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:17.941 UTC [4538] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:17.953 UTC [4538] LOG: database system is shut down logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.435 UTC [4608] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.435 UTC [4608] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.435 UTC [4608] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.462 UTC [4608] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.471 UTC [4613] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.502 UTC [4613] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.531 UTC [4613] LOG: entering standby mode logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.562 UTC [4613] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.886 UTC [4613] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.922 UTC [4613] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.924 UTC [4608] LOG: startup process (PID 4613) exited with exit code 1 logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.924 UTC [4608] LOG: aborting startup due to startup process failure logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.924 UTC [4608] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.937 UTC [4608] LOG: database system is shut down logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.445 UTC [4653] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.446 UTC [4653] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.446 UTC [4653] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.473 UTC [4653] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.483 UTC [4658] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.518 UTC [4658] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.549 UTC [4658] LOG: entering standby mode logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.579 UTC [4658] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.903 UTC [4658] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.939 UTC [4658] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.941 UTC [4653] LOG: startup process (PID 4658) exited with exit code 1 logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.941 UTC [4653] LOG: aborting startup due to startup process failure logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.941 UTC [4653] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:51 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.953 UTC [4653] LOG: database system is shut down logger.go:42: 19:47:52 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:17.793 UTC [4588] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:17.802 UTC [4588] LOG: database system is shut down logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.328 UTC [4657] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.328 UTC [4657] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.328 UTC [4657] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.355 UTC [4657] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.364 UTC [4671] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.393 UTC [4671] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.420 UTC [4671] LOG: entering standby mode logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.448 UTC [4671] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.772 UTC [4671] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.807 UTC [4671] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.808 UTC [4657] LOG: startup process (PID 4671) exited with exit code 1 logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.808 UTC [4657] LOG: aborting startup due to startup process failure logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.808 UTC [4657] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:37.816 UTC [4657] LOG: database system is shut down logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.320 UTC [4701] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.320 UTC [4701] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.320 UTC [4701] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.346 UTC [4701] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.355 UTC [4715] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.381 UTC [4715] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.411 UTC [4715] LOG: entering standby mode logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.437 UTC [4715] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.759 UTC [4715] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.793 UTC [4715] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.795 UTC [4701] LOG: startup process (PID 4715) exited with exit code 1 logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.795 UTC [4701] LOG: aborting startup due to startup process failure logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.795 UTC [4701] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:47:47.802 UTC [4701] LOG: database system is shut down logger.go:42: 19:47:53 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:47:55 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:48:26 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 24m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 24m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 25m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 21m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 21m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 20m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 20m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 20m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 24m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 24m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 24m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 23m logger.go:42: 19:48:27 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 26m logger.go:42: 19:48:28 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:07.981 UTC [4729] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:07.993 UTC [4729] LOG: database system is shut down logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.468 UTC [4773] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.469 UTC [4773] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.469 UTC [4773] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.495 UTC [4773] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.507 UTC [4778] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.539 UTC [4778] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.575 UTC [4778] LOG: entering standby mode logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.624 UTC [4778] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.954 UTC [4778] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.990 UTC [4778] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.992 UTC [4773] LOG: startup process (PID 4778) exited with exit code 1 logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.993 UTC [4773] LOG: aborting startup due to startup process failure logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.993 UTC [4773] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:18.005 UTC [4773] LOG: database system is shut down logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.809 UTC [4817] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.810 UTC [4817] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.810 UTC [4817] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.836 UTC [4817] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.846 UTC [4822] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.879 UTC [4822] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.910 UTC [4822] LOG: entering standby mode logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.943 UTC [4822] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.269 UTC [4822] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.306 UTC [4822] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.308 UTC [4817] LOG: startup process (PID 4822) exited with exit code 1 logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.308 UTC [4817] LOG: aborting startup due to startup process failure logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.308 UTC [4817] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:29 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.321 UTC [4817] LOG: database system is shut down logger.go:42: 19:48:30 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:07.776 UTC [4778] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:07.784 UTC [4778] LOG: database system is shut down logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.342 UTC [4822] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.343 UTC [4822] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.343 UTC [4822] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.369 UTC [4822] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.378 UTC [4836] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.406 UTC [4836] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.433 UTC [4836] LOG: entering standby mode logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.459 UTC [4836] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.781 UTC [4836] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.816 UTC [4836] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.818 UTC [4822] LOG: startup process (PID 4836) exited with exit code 1 logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.818 UTC [4822] LOG: aborting startup due to startup process failure logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.818 UTC [4822] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:17.826 UTC [4822] LOG: database system is shut down logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.719 UTC [4874] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.719 UTC [4874] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.719 UTC [4874] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.745 UTC [4874] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.753 UTC [4879] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.780 UTC [4879] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.806 UTC [4879] LOG: entering standby mode logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:27.831 UTC [4879] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.153 UTC [4879] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.188 UTC [4879] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.189 UTC [4874] LOG: startup process (PID 4879) exited with exit code 1 logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.189 UTC [4874] LOG: aborting startup due to startup process failure logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.189 UTC [4874] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:31 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.198 UTC [4874] LOG: database system is shut down logger.go:42: 19:48:32 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:48:33 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:49:04 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 25m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 25m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 25m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 22m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 22m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 20m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 20m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 21m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 25m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 25m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 25m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 24m logger.go:42: 19:49:05 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 27m logger.go:42: 19:49:06 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.308 UTC [4817] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.321 UTC [4817] LOG: database system is shut down logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.491 UTC [4903] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.492 UTC [4903] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.492 UTC [4903] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.519 UTC [4903] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.528 UTC [4908] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.563 UTC [4908] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.598 UTC [4908] LOG: entering standby mode logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.644 UTC [4908] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.973 UTC [4908] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:48.010 UTC [4908] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:48.012 UTC [4903] LOG: startup process (PID 4908) exited with exit code 1 logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:48.012 UTC [4903] LOG: aborting startup due to startup process failure logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:48.012 UTC [4903] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:48.023 UTC [4903] LOG: database system is shut down logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.503 UTC [4971] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.504 UTC [4971] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.504 UTC [4971] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.530 UTC [4971] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.540 UTC [4976] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.571 UTC [4976] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.599 UTC [4976] LOG: entering standby mode logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.632 UTC [4976] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.961 UTC [4976] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.998 UTC [4976] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:08.000 UTC [4971] LOG: startup process (PID 4976) exited with exit code 1 logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:08.000 UTC [4971] LOG: aborting startup due to startup process failure logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:08.000 UTC [4971] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:08.013 UTC [4971] LOG: database system is shut down logger.go:42: 19:49:08 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.189 UTC [4874] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:28.198 UTC [4874] LOG: database system is shut down logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.306 UTC [4951] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.307 UTC [4951] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.307 UTC [4951] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.333 UTC [4951] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.341 UTC [4956] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.368 UTC [4956] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.393 UTC [4956] LOG: entering standby mode logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.420 UTC [4956] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.743 UTC [4956] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.778 UTC [4956] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.779 UTC [4951] LOG: startup process (PID 4956) exited with exit code 1 logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.779 UTC [4951] LOG: aborting startup due to startup process failure logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.779 UTC [4951] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:47.788 UTC [4951] LOG: database system is shut down logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.320 UTC [5020] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.321 UTC [5020] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.321 UTC [5020] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.346 UTC [5020] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.355 UTC [5025] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.384 UTC [5025] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.410 UTC [5025] LOG: entering standby mode logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.436 UTC [5025] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.758 UTC [5025] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.792 UTC [5025] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.794 UTC [5020] LOG: startup process (PID 5025) exited with exit code 1 logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.794 UTC [5020] LOG: aborting startup due to startup process failure logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.794 UTC [5020] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.802 UTC [5020] LOG: database system is shut down logger.go:42: 19:49:10 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:49:11 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 26m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 26m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 26m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 22m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 23m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 21m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 21m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 21m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 25m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 25m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 25m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 24m logger.go:42: 19:49:43 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 28m logger.go:42: 19:49:45 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:48.012 UTC [4903] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:48:48.023 UTC [4903] LOG: database system is shut down logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.503 UTC [4971] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.504 UTC [4971] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.504 UTC [4971] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.530 UTC [4971] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.540 UTC [4976] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.571 UTC [4976] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.599 UTC [4976] LOG: entering standby mode logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.632 UTC [4976] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.961 UTC [4976] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.998 UTC [4976] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:08.000 UTC [4971] LOG: startup process (PID 4976) exited with exit code 1 logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:08.000 UTC [4971] LOG: aborting startup due to startup process failure logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:08.000 UTC [4971] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:08.013 UTC [4971] LOG: database system is shut down logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.510 UTC [5047] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.511 UTC [5047] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.511 UTC [5047] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.537 UTC [5047] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.547 UTC [5052] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.582 UTC [5052] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.615 UTC [5052] LOG: entering standby mode logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.649 UTC [5052] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.981 UTC [5052] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:28.020 UTC [5052] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:28.022 UTC [5047] LOG: startup process (PID 5052) exited with exit code 1 logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:28.022 UTC [5047] LOG: aborting startup due to startup process failure logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:28.022 UTC [5047] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:46 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:28.041 UTC [5047] LOG: database system is shut down logger.go:42: 19:49:47 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.794 UTC [5020] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:07.802 UTC [5020] LOG: database system is shut down logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.316 UTC [5097] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.317 UTC [5097] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.317 UTC [5097] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.343 UTC [5097] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.352 UTC [5102] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.381 UTC [5102] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.409 UTC [5102] LOG: entering standby mode logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.436 UTC [5102] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.758 UTC [5102] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.792 UTC [5102] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.794 UTC [5097] LOG: startup process (PID 5102) exited with exit code 1 logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.794 UTC [5097] LOG: aborting startup due to startup process failure logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.794 UTC [5097] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.802 UTC [5097] LOG: database system is shut down logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.306 UTC [5164] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.307 UTC [5164] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.307 UTC [5164] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.333 UTC [5164] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.342 UTC [5169] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.371 UTC [5169] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.398 UTC [5169] LOG: entering standby mode logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.425 UTC [5169] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.748 UTC [5169] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.781 UTC [5169] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.783 UTC [5164] LOG: startup process (PID 5169) exited with exit code 1 logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.783 UTC [5164] LOG: aborting startup due to startup process failure logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.783 UTC [5164] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:48 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.791 UTC [5164] LOG: database system is shut down logger.go:42: 19:49:49 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:49:50 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:50:21 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 26m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 26m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 27m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 23m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 23m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 22m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 22m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 22m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 26m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 26m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 26m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 25m logger.go:42: 19:50:22 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 28m logger.go:42: 19:50:23 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:28.022 UTC [5047] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:28.041 UTC [5047] LOG: database system is shut down logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.468 UTC [5123] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.469 UTC [5123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.469 UTC [5123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.497 UTC [5123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.507 UTC [5128] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.543 UTC [5128] LOG: restored log file "00000003.history" from archive logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.572 UTC [5128] LOG: entering standby mode logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.608 UTC [5128] LOG: restored log file "00000003.history" from archive logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.944 UTC [5128] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.982 UTC [5128] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.984 UTC [5123] LOG: startup process (PID 5128) exited with exit code 1 logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.984 UTC [5123] LOG: aborting startup due to startup process failure logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.984 UTC [5123] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.996 UTC [5123] LOG: database system is shut down logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.440 UTC [5193] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.441 UTC [5193] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.441 UTC [5193] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.466 UTC [5193] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.475 UTC [5198] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.505 UTC [5198] LOG: restored log file "00000003.history" from archive logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.533 UTC [5198] LOG: entering standby mode logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.561 UTC [5198] LOG: restored log file "00000003.history" from archive logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.884 UTC [5198] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.921 UTC [5198] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.922 UTC [5193] LOG: startup process (PID 5198) exited with exit code 1 logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.922 UTC [5193] LOG: aborting startup due to startup process failure logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.922 UTC [5193] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:50:24 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.935 UTC [5193] LOG: database system is shut down logger.go:42: 19:50:25 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.794 UTC [5097] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:27.802 UTC [5097] LOG: database system is shut down logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.306 UTC [5164] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.307 UTC [5164] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.307 UTC [5164] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.333 UTC [5164] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.342 UTC [5169] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.371 UTC [5169] LOG: restored log file "00000003.history" from archive logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.398 UTC [5169] LOG: entering standby mode logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.425 UTC [5169] LOG: restored log file "00000003.history" from archive logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.748 UTC [5169] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.781 UTC [5169] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.783 UTC [5164] LOG: startup process (PID 5169) exited with exit code 1 logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.783 UTC [5164] LOG: aborting startup due to startup process failure logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.783 UTC [5164] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:49:47.791 UTC [5164] LOG: database system is shut down logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.298 UTC [5242] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.299 UTC [5242] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.299 UTC [5242] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.324 UTC [5242] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.332 UTC [5247] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.358 UTC [5247] LOG: restored log file "00000003.history" from archive logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.383 UTC [5247] LOG: entering standby mode logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.409 UTC [5247] LOG: restored log file "00000003.history" from archive logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.731 UTC [5247] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.765 UTC [5247] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.767 UTC [5242] LOG: startup process (PID 5247) exited with exit code 1 logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.767 UTC [5242] LOG: aborting startup due to startup process failure logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.767 UTC [5242] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:50:26 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:07.775 UTC [5242] LOG: database system is shut down logger.go:42: 19:50:27 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:50:28 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:50:59 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 27m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 27m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 27m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 24m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 24m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 22m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 22m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 22m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 27m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 27m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 27m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 26m logger.go:42: 19:51:00 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 29m logger.go:42: 19:51:02 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:27.912 UTC [5268] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:27.923 UTC [5268] LOG: database system is shut down logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.430 UTC [5312] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.431 UTC [5312] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.431 UTC [5312] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.456 UTC [5312] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.465 UTC [5317] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.499 UTC [5317] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.533 UTC [5317] LOG: entering standby mode logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.563 UTC [5317] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.886 UTC [5317] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.921 UTC [5317] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.923 UTC [5312] LOG: startup process (PID 5317) exited with exit code 1 logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.923 UTC [5312] LOG: aborting startup due to startup process failure logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.923 UTC [5312] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:37.934 UTC [5312] LOG: database system is shut down logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.449 UTC [5380] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.449 UTC [5380] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.449 UTC [5380] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.475 UTC [5380] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.483 UTC [5385] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.513 UTC [5385] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.542 UTC [5385] LOG: entering standby mode logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.571 UTC [5385] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.894 UTC [5385] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.930 UTC [5385] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.931 UTC [5380] LOG: startup process (PID 5385) exited with exit code 1 logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.931 UTC [5380] LOG: aborting startup due to startup process failure logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.931 UTC [5380] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.943 UTC [5380] LOG: database system is shut down logger.go:42: 19:51:03 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:27.814 UTC [5319] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:27.822 UTC [5319] LOG: database system is shut down logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.290 UTC [5388] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.291 UTC [5388] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.291 UTC [5388] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.317 UTC [5388] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.325 UTC [5393] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.351 UTC [5393] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.378 UTC [5393] LOG: entering standby mode logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.404 UTC [5393] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.727 UTC [5393] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.762 UTC [5393] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.763 UTC [5388] LOG: startup process (PID 5393) exited with exit code 1 logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.763 UTC [5388] LOG: aborting startup due to startup process failure logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.763 UTC [5388] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:47.771 UTC [5388] LOG: database system is shut down logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.316 UTC [5432] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.316 UTC [5432] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.316 UTC [5432] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.342 UTC [5432] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.350 UTC [5437] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.377 UTC [5437] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.403 UTC [5437] LOG: entering standby mode logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.429 UTC [5437] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.751 UTC [5437] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.784 UTC [5437] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.786 UTC [5432] LOG: startup process (PID 5437) exited with exit code 1 logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.786 UTC [5432] LOG: aborting startup due to startup process failure logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.786 UTC [5432] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.794 UTC [5432] LOG: database system is shut down logger.go:42: 19:51:05 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:51:06 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:51:38 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 27m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 27m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 28m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 24m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 25m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 23m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 23m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 23m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 27m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 27m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 27m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 26m logger.go:42: 19:51:39 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 30m logger.go:42: 19:51:40 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.931 UTC [5380] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.943 UTC [5380] LOG: database system is shut down logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.429 UTC [5465] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.430 UTC [5465] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.430 UTC [5465] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.456 UTC [5465] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.465 UTC [5470] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.497 UTC [5470] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.525 UTC [5470] LOG: entering standby mode logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.554 UTC [5470] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.878 UTC [5470] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.915 UTC [5470] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.917 UTC [5465] LOG: startup process (PID 5470) exited with exit code 1 logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.917 UTC [5465] LOG: aborting startup due to startup process failure logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.917 UTC [5465] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:17.929 UTC [5465] LOG: database system is shut down logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.438 UTC [5508] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.438 UTC [5508] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.438 UTC [5508] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.464 UTC [5508] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.473 UTC [5513] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.504 UTC [5513] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.533 UTC [5513] LOG: entering standby mode logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.564 UTC [5513] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.888 UTC [5513] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.924 UTC [5513] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.925 UTC [5508] LOG: startup process (PID 5513) exited with exit code 1 logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.925 UTC [5508] LOG: aborting startup due to startup process failure logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.925 UTC [5508] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:41 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.937 UTC [5508] LOG: database system is shut down logger.go:42: 19:51:42 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.786 UTC [5432] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:50:57.794 UTC [5432] LOG: database system is shut down logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.311 UTC [5484] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.311 UTC [5484] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.311 UTC [5484] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.338 UTC [5484] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.346 UTC [5489] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.373 UTC [5489] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.399 UTC [5489] LOG: entering standby mode logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.427 UTC [5489] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.748 UTC [5489] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.782 UTC [5489] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.784 UTC [5484] LOG: startup process (PID 5489) exited with exit code 1 logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.784 UTC [5484] LOG: aborting startup due to startup process failure logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.784 UTC [5484] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:07.792 UTC [5484] LOG: database system is shut down logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.305 UTC [5552] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.306 UTC [5552] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.306 UTC [5552] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.332 UTC [5552] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.340 UTC [5557] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.367 UTC [5557] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.393 UTC [5557] LOG: entering standby mode logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.419 UTC [5557] LOG: restored log file "00000003.history" from archive logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.741 UTC [5557] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.776 UTC [5557] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.778 UTC [5552] LOG: startup process (PID 5557) exited with exit code 1 logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.778 UTC [5552] LOG: aborting startup due to startup process failure logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.778 UTC [5552] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:51:43 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.786 UTC [5552] LOG: database system is shut down logger.go:42: 19:51:44 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:51:45 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:52:16 | major-upgrade-15-to-16/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz 0/1 Completed 0 28m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr 0/1 Completed 0 28m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | 15-to-16-pgdata-9rhjd 0/1 Completed 0 29m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-6hhz-zw66b 0/1 Completed 0 25m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-backup-h8qp-bnljp 0/1 Completed 0 25m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-gfw2-0 3/4 Running 0 23m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-jcr4-0 3/4 Running 0 23m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-instance1-pzj7-0 4/4 Running 0 24m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 2/2 Running 0 28m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj 2/2 Running 0 28m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs 2/2 Running 0 28m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | major-upgrade-15-to-16-repo-host-0 2/2 Running 0 27m logger.go:42: 19:52:17 | major-upgrade-15-to-16/10-run-restore | pg-client-65d98588cc-qqkfq 1/1 Running 0 30m logger.go:42: 19:52:18 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-gfw2-0: logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.907 UTC [5585] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.919 UTC [5585] LOG: database system is shut down logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.453 UTC [5629] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.453 UTC [5629] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.453 UTC [5629] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.479 UTC [5629] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.488 UTC [5634] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.519 UTC [5634] LOG: restored log file "00000003.history" from archive logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.550 UTC [5634] LOG: entering standby mode logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.580 UTC [5634] LOG: restored log file "00000003.history" from archive logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.903 UTC [5634] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.938 UTC [5634] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.939 UTC [5629] LOG: startup process (PID 5634) exited with exit code 1 logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.939 UTC [5629] LOG: aborting startup due to startup process failure logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.939 UTC [5629] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:57.951 UTC [5629] LOG: database system is shut down logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.467 UTC [5697] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.468 UTC [5697] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.468 UTC [5697] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.494 UTC [5697] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.505 UTC [5702] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.539 UTC [5702] LOG: restored log file "00000003.history" from archive logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.569 UTC [5702] LOG: entering standby mode logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.603 UTC [5702] LOG: restored log file "00000003.history" from archive logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.934 UTC [5702] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.971 UTC [5702] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.972 UTC [5697] LOG: startup process (PID 5702) exited with exit code 1 logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.973 UTC [5697] LOG: aborting startup due to startup process failure logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.973 UTC [5697] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:52:19 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:17.985 UTC [5697] LOG: database system is shut down logger.go:42: 19:52:20 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-jcr4-0: logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.778 UTC [5552] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:27.786 UTC [5552] LOG: database system is shut down logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.303 UTC [5629] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.303 UTC [5629] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.303 UTC [5629] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.329 UTC [5629] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.338 UTC [5634] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.364 UTC [5634] LOG: restored log file "00000003.history" from archive logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.390 UTC [5634] LOG: entering standby mode logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.415 UTC [5634] LOG: restored log file "00000003.history" from archive logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.737 UTC [5634] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.771 UTC [5634] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.772 UTC [5629] LOG: startup process (PID 5634) exited with exit code 1 logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.772 UTC [5629] LOG: aborting startup due to startup process failure logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.772 UTC [5629] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:51:47.780 UTC [5629] LOG: database system is shut down logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.321 UTC [5698] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.321 UTC [5698] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.322 UTC [5698] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.348 UTC [5698] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.356 UTC [5703] LOG: database system was shut down in recovery at 2026-02-20 19:27:29 UTC logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.383 UTC [5703] LOG: restored log file "00000003.history" from archive logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.409 UTC [5703] LOG: entering standby mode logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.438 UTC [5703] LOG: restored log file "00000003.history" from archive logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.760 UTC [5703] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.794 UTC [5703] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006648 on timeline 2 logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.796 UTC [5698] LOG: startup process (PID 5703) exited with exit code 1 logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.796 UTC [5698] LOG: aborting startup due to startup process failure logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.796 UTC [5698] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:52:21 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:52:07.804 UTC [5698] LOG: database system is shut down logger.go:42: 19:52:22 | major-upgrade-15-to-16/10-run-restore | PostgreSQL logs from major-upgrade-15-to-16-instance1-pzj7-0: logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:22.817 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:23.484 UTC [135] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.602 s, sync=0.007 s, total=0.667 s; sync files=5, longest=0.006 s, average=0.002 s; distance=16383 kB, estimate=16383 kB; lsn=0/B000060, redo lsn=0/B000028 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.211 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:34.281 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.070 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/C000060, redo lsn=0/C000028 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.120 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:24:35.181 UTC [135] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.061 s; sync files=0, longest=0.000 s, average=0.000 s; distance=16384 kB, estimate=16384 kB; lsn=0/D000060, redo lsn=0/D000028 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:32.045 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.302 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.615 UTC [135] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.515 s, sync=0.006 s, total=3.571 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB; lsn=0/F000060, redo lsn=0/F000028 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:35.616 UTC [1349] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.125 UTC [135] LOG: checkpoint starting: force wait logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:26:53.504 UTC [135] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.309 s, sync=0.005 s, total=0.379 s; sync files=5, longest=0.003 s, average=0.001 s; distance=49152 kB, estimate=49152 kB; lsn=0/12000060, redo lsn=0/12000028 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.600 UTC [124] LOG: starting PostgreSQL 16.11 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.601 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.627 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.643 UTC [129] LOG: database system was shut down at 2026-02-20 19:27:59 UTC logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.666 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:14.734 UTC [133] LOG: pg_cron scheduler started logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:28:15.524 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:14.714 UTC [127] LOG: checkpoint starting: time logger.go:42: 19:52:23 | major-upgrade-15-to-16/10-run-restore | 2026-02-20 19:33:19.261 UTC [127] LOG: checkpoint complete: wrote 49 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.524 s, sync=0.006 s, total=4.548 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB; lsn=0/14000060, redo lsn=0/14000028 logger.go:42: 19:52:26 | major-upgrade-15-to-16/10-run-restore | test step failed 10-run-restore logger.go:42: 19:52:26 | major-upgrade-15-to-16/10-run-restore | collecting log output for [type==pod,label: postgres-operator.crunchydata.com/data=postgres] logger.go:42: 19:52:26 | major-upgrade-15-to-16/10-run-restore | running command: [kubectl logs --prefix -l postgres-operator.crunchydata.com/data=postgres -n kuttl-test-equipped-lab --all-containers --tail=30] logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] End-of-backup record required: no logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] wal_level setting: logical logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] wal_log_hints setting: on logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] max_connections setting: 100 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] max_worker_processes setting: 8 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] max_wal_senders setting: 10 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] max_prepared_xacts setting: 0 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] max_locks_per_xact setting: 64 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] track_commit_timestamp setting: on logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Maximum data alignment: 8 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Database block size: 8192 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Blocks per segment of large relation: 131072 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] WAL block size: 8192 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Bytes per WAL segment: 16777216 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Maximum length of identifiers: 64 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Maximum columns in an index: 32 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Maximum size of a TOAST chunk: 1996 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Size of a large-object chunk: 2048 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Date/time type storage: 64-bit integers logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Float8 argument passing: by value logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Data page checksum version: 1 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] Mock authentication nonce: 3056a7f1797f01ebf7c1cd7c7a18fe91058f27210897918aaa5c6fbf0b416fea logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] 2026-02-20 19:52:26,564 INFO: Lock owner: major-upgrade-15-to-16-instance1-pzj7-0; I am major-upgrade-15-to-16-instance1-jcr4-0 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] 2026-02-20 19:52:26,565 INFO: starting as a secondary logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] 2026-02-20 19:52:27.077 UTC [5782] LOG: pgaudit extension initialized logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] 2026-02-20 19:52:27,105 INFO: postmaster pid=5782 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] /tmp/postgres:5432 - no response logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] 2026-02-20 19:52:27.318 UTC [5782] LOG: redirecting log output to logging collector process logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database] 2026-02-20 19:52:27.318 UTC [5782] HINT: Future log output will appear in directory "log". logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] Initializing ... logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-16/bin/postgres logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 16.11 - Percona Distribution logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg16 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg16 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: data version::16 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg16_wal logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -u logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -g logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -u logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -g logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -u logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -g logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -u logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -g logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -u logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -g logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -u logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] ++ id -g logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] Initializing ... logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-16/bin/postgres logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 16.11 - Percona Distribution logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg16 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg16 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: data version::16 logger.go:42: 19:52:27 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg16_wal logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] Initializing ... logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-16/bin/postgres logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 16.11 - Percona Distribution logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg16 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg16 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: bootstrap directory::/pgdata/pg16_bootstrap logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: data version::16 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg16_wal logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] wal_level setting: logical logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] wal_log_hints setting: on logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] max_connections setting: 100 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] max_worker_processes setting: 8 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] max_wal_senders setting: 10 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] max_prepared_xacts setting: 0 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] max_locks_per_xact setting: 64 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] track_commit_timestamp setting: on logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Maximum data alignment: 8 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Database block size: 8192 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Blocks per segment of large relation: 131072 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] WAL block size: 8192 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Bytes per WAL segment: 16777216 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Maximum length of identifiers: 64 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Maximum columns in an index: 32 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Maximum size of a TOAST chunk: 1996 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Size of a large-object chunk: 2048 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Date/time type storage: 64-bit integers logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Float8 argument passing: by value logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Data page checksum version: 1 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] Mock authentication nonce: 3056a7f1797f01ebf7c1cd7c7a18fe91058f27210897918aaa5c6fbf0b416fea logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] 2026-02-20 19:52:26,568 INFO: Lock owner: major-upgrade-15-to-16-instance1-pzj7-0; I am major-upgrade-15-to-16-instance1-gfw2-0 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] 2026-02-20 19:52:26,570 INFO: starting as a secondary logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] 2026-02-20 19:52:27.225 UTC [5749] LOG: pgaudit extension initialized logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] 2026-02-20 19:52:27,251 INFO: postmaster pid=5749 logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] /tmp/postgres:5432 - no response logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] 2026-02-20 19:52:27.520 UTC [5749] LOG: redirecting log output to logging collector process logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] 2026-02-20 19:52:27.520 UTC [5749] HINT: Future log output will appear in directory "log". logger.go:42: 19:52:28 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database] 2026-02-20 19:52:28,266 ERROR: postmaster is not running logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:47:46,555 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:47:56,549 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:48:06,552 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:48:16,563 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:48:26,548 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:48:36,551 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:48:46,551 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:48:56,550 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:49:06,551 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:49:16,549 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:49:16,781 ERROR: ObjectCache.run ProtocolError("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:49:26,550 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:49:36,550 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:49:46,553 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:49:56,549 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:50:06,554 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:50:16,551 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:50:26,549 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:50:36,550 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:50:46,551 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:50:56,552 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:51:06,551 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:51:16,549 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:51:26,553 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:51:36,550 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:51:46,552 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:51:56,552 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:52:06,552 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:52:16,548 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database] 2026-02-20 19:52:26,549 INFO: no action. I am (major-upgrade-15-to-16-instance1-pzj7-0), the leader with the lock logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=106-50fa77c7 --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@7ef15c3b-ce2b-4e31-81cb-bd42acf707dd=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:52:29 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + IFS=, logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + read -ra extensions logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/16') logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + declare -a args logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + [[ -n '' ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + [[ '' == \t\r\u\e ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + [[ '' == \t\r\u\e ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] ++ basename -- /pgdata/extension/16/pg_cron-pg16-1.6.6.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + filename=pg_cron-pg16-1.6.6.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + key=pg_cron-pg16-1.6.6 logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + [[ pg_cron-pg16-1.6.6 == \* ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + [[ ! pg_cron-pg16-1.6.6 =~ pg_cron-pg16-1.6.6 ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + for key in "${extensions[@]}" logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + '[' -f /pgdata/extension/16/pg_cron-pg16-1.6.6.installed ']' logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + echo 'Extension pg_cron-pg16-1.6.6 already installed' logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] + continue logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-pzj7-0/extension-installer-16] Extension pg_cron-pg16-1.6.6 already installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=112-eda84b41 --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@7ef15c3b-ce2b-4e31-81cb-bd42acf707dd=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + IFS=, logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + read -ra extensions logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/16') logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + declare -a args logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + [[ -n '' ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + [[ '' == \t\r\u\e ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + [[ '' == \t\r\u\e ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] ++ basename -- /pgdata/extension/16/pg_cron-pg16-1.6.6.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + filename=pg_cron-pg16-1.6.6.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + key=pg_cron-pg16-1.6.6 logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + [[ pg_cron-pg16-1.6.6 == \* ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + [[ ! pg_cron-pg16-1.6.6 =~ pg_cron-pg16-1.6.6 ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + for key in "${extensions[@]}" logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + '[' -f /pgdata/extension/16/pg_cron-pg16-1.6.6.installed ']' logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + echo 'Extension pg_cron-pg16-1.6.6 already installed' logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] + continue logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/extension-installer-16] Extension pg_cron-pg16-1.6.6 already installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -u logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] ++ id -g logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-gfw2-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=110-d43e5079 --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@7ef15c3b-ce2b-4e31-81cb-bd42acf707dd=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + IFS=, logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + read -ra extensions logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/16') logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + declare -a args logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + [[ -n '' ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + [[ '' == \t\r\u\e ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + [[ '' == \t\r\u\e ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] ++ basename -- /pgdata/extension/16/pg_cron-pg16-1.6.6.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + filename=pg_cron-pg16-1.6.6.installed logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + key=pg_cron-pg16-1.6.6 logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + [[ pg_cron-pg16-1.6.6 == \* ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + [[ ! pg_cron-pg16-1.6.6 =~ pg_cron-pg16-1.6.6 ]] logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + for key in "${extensions[@]}" logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + '[' -f /pgdata/extension/16/pg_cron-pg16-1.6.6.installed ']' logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + echo 'Extension pg_cron-pg16-1.6.6 already installed' logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] + continue logger.go:42: 19:52:30 | major-upgrade-15-to-16/10-run-restore | [pod/major-upgrade-15-to-16-instance1-jcr4-0/extension-installer-16] Extension pg_cron-pg16-1.6.6 already installed case.go:328: failed in step 10-run-restore case.go:330: --- PerconaPGCluster:kuttl-test-equipped-lab/major-upgrade-15-to-16 +++ PerconaPGCluster:kuttl-test-equipped-lab/major-upgrade-15-to-16 @@ -1,18 +1,41 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"pgv2.percona.com/v2","kind":"PerconaPGCluster","metadata":{"annotations":{},"labels":{"e2e":"major-upgrade-15-to-16"},"name":"major-upgrade-15-to-16","namespace":"kuttl-test-equipped-lab"},"spec":{"backups":{"pgbackrest":{"image":"perconalab/percona-postgresql-operator:main-pgbackrest15","manual":{"options":["--type=full"],"repoName":"repo1"},"repoHost":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/data":"pgbackrest"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}}},"repos":[{"name":"repo1","schedules":{"full":"0 0 * * 6"},"volume":{"volumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}}}}]}},"crVersion":"2.9.0","extensions":{"custom":[{"name":"pg_cron","version":"1.6.6"}],"image":"docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0","imagePullPolicy":"Always","storage":{"bucket":"pg-extensions","region":"eu-central-1","secret":{"name":"aws-s3-secret"},"type":"s3"}},"image":"perconalab/percona-postgresql-operator:main-ppg15-postgres","imagePullPolicy":"Always","initContainer":{"image":"docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0"},"instances":[{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/data":"postgres"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"3Gi"}}},"name":"instance1","replicas":3}],"patroni":{"dynamicConfiguration":{"postgresql":{"parameters":{"shared_preload_libraries":"pg_cron"}}}},"pmm":{"customClusterName":"major-upgrade-15-to-16-pmm-custom-name","enabled":false,"image":"docker.io/perconalab/pmm-client:dev-latest","postgresParams":"--environment=dev-postgres","secret":"major-upgrade-15-to-16-pmm-secret","serverHost":"monitoring-service"},"postgresVersion":15,"proxy":{"pgBouncer":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/role":"pgbouncer"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}},"image":"perconalab/percona-postgresql-operator:main-pgbouncer15","replicas":3}},"users":[{"name":"postgres","password":{"type":"AlphaNumeric"}},{"name":"major-upgrade-15-to-16","password":{"type":"AlphaNumeric"}}]}} + pgv2.percona.com/patroni-version: 4.1.0 + finalizers: + - internal.percona.com/stop-watchers + labels: + e2e: major-upgrade-15-to-16 + managedFields: '[... elided field over 10 lines long ...]' name: major-upgrade-15-to-16 namespace: kuttl-test-equipped-lab +spec: '[... elided field over 10 lines long ...]' status: + conditions: '[... elided field over 10 lines long ...]' + host: major-upgrade-15-to-16-pgbouncer.kuttl-test-equipped-lab.svc + installedCustomExtensions: + - pg_cron + observedGeneration: 8 + patroni: + status: + systemIdentifier: "7609030249273004126" + version: 4.1.0 + patroniVersion: 4.1.0 + pgbackrest: '[... elided field over 10 lines long ...]' pgbouncer: ready: 3 size: 3 postgres: + imageID: docker.io/perconalab/percona-postgresql-operator@sha256:497cac9e970eeaf42ce17f37f289398969554c6aa5875048ac20cf486021fd95 instances: - name: instance1 - ready: 3 + ready: 1 size: 3 - ready: 3 + ready: 1 size: 3 - state: ready + version: 16 + state: initializing case.go:330: resource PerconaPGCluster:kuttl-test-equipped-lab/major-upgrade-15-to-16: .status.postgres.ready: value mismatch, expected: 3 != actual: 1 case.go:330: command "kubectl -n ${NAMESPACE} get pod\\n for pod in $(kubectl -n ${NAMESPA..." exceeded 10 sec timeout, context deadline exceeded logger.go:42: 19:52:31 | major-upgrade-15-to-16 | major-upgrade-15-to-16 events from ns kuttl-test-equipped-lab: logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:27 +0000 UTC Normal Pod pg-client-65d98588cc-qqkfq Binding Scheduled Successfully assigned kuttl-test-equipped-lab/pg-client-65d98588cc-qqkfq to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:27 +0000 UTC Normal ReplicaSet.apps pg-client-65d98588cc SuccessfulCreate Created pod: pg-client-65d98588cc-qqkfq replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:27 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-65d98588cc from 0 to 1 deployment-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:28 +0000 UTC Normal Pod pg-client-65d98588cc-qqkfq.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:28 +0000 UTC Normal Pod pg-client-65d98588cc-qqkfq.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 185ms (185ms including waiting). Image size: 564266549 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:28 +0000 UTC Normal Pod pg-client-65d98588cc-qqkfq.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:28 +0000 UTC Normal Pod pg-client-65d98588cc-qqkfq.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:33 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-pzj7-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-gfw2-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-gfw2-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-gfw2-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-gfw2-pgdata" pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-gfw2 SuccessfulCreate create Pod major-upgrade-15-to-16-instance1-gfw2-0 in StatefulSet major-upgrade-15-to-16-instance1-gfw2 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-jcr4-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-jcr4-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-jcr4-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-jcr4-pgdata" pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-jcr4 SuccessfulCreate create Pod major-upgrade-15-to-16-instance1-jcr4-0 in StatefulSet major-upgrade-15-to-16-instance1-jcr4 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-pzj7-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-pzj7-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-pzj7-pgdata" pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:34 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-pzj7 SuccessfulCreate create Pod major-upgrade-15-to-16-instance1-pzj7-0 in StatefulSet major-upgrade-15-to-16-instance1-pzj7 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-7g2j default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-588fd95b56 SuccessfulCreate Created pod: major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-588fd95b56 SuccessfulCreate Created pod: major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-588fd95b56 SuccessfulCreate Created pod: major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal Deployment.apps major-upgrade-15-to-16-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-15-to-16-pgbouncer-588fd95b56 from 0 to 3 deployment-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-15-to-16-pgbouncer NoPods No matching pods found controllermanager logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-repo-host SuccessfulCreate create Pod major-upgrade-15-to-16-repo-host-0 in StatefulSet major-upgrade-15-to-16-repo-host successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-repo1 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-equipped-lab/major-upgrade-15-to-16-repo1" pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:35 +0000 UTC Normal PostgresCluster.postgres-operator.crunchydata.com major-upgrade-15-to-16 RepoHostCreated created pgBackRest repository host StatefulSet/major-upgrade-15-to-16-repo-host postgrescluster-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer15" in 174ms (174ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer15" in 213ms (213ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer15" in 230ms (230ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer15" in 242ms (242ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer15" in 241ms (241ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer15" in 235ms (235ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:36 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:37 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:38 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-gfw2-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:38 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-gfw2-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-e6ab99f1-3a44-4a59-9ed0-17c17add560d pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:38 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-jcr4-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-7g2j default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:38 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-jcr4-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-d0638d02-35e5-4b96-af8d-12af88b0fb04 pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:38 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-pzj7-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:38 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-instance1-pzj7-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-28d721ca-b172-4140-87fb-5e5a8c0bc85b pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:38 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-ff1c4cb0-8e5b-40c3-b2c7-28bc4e8b7861 pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:39 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:45 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-28d721ca-b172-4140-87fb-5e5a8c0bc85b" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:46 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-e6ab99f1-3a44-4a59-9ed0-17c17add560d" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:46 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-d0638d02-35e5-4b96-af8d-12af88b0fb04" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:46 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-ff1c4cb0-8e5b-40c3-b2c7-28bc4e8b7861" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:47 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:47 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 231ms (231ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:47 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:47 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 191ms (191ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest15" in 241ms (241ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest15" in 220ms (220ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:48 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 272ms (272ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 306ms (306ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest15" in 306ms (306ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:49 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 266ms (266ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 226ms (226ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest15" in 211ms (211ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:50 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 220ms (220ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 264ms (264ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:51 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 237ms (237ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 234ms (234ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 225ms (225ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:52 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 229ms (229ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 253ms (253ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 220ms (220ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:53 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:54 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:54 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:55 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:55 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 209ms (209ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:55 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:55 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:55 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:55 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 227ms (227ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:55 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:55 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 223ms (223ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg15-postgres" in 213ms (213ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:56 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:59 +0000 UTC Warning Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:21:59 +0000 UTC Warning Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:01 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-backup-vpgb-plggw to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:01 +0000 UTC Normal Job.batch major-upgrade-15-to-16-backup-vpgb SuccessfulCreate Created pod: major-upgrade-15-to-16-backup-vpgb-plggw job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:01 +0000 UTC Normal PostgresCluster.postgres-operator.crunchydata.com major-upgrade-15-to-16 StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:02 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:02 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 176ms (176ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:02 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:02 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:03 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw.spec.containers{pgbackrest} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:03 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw.spec.containers{pgbackrest} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest15" in 298ms (298ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:03 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:03 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-vpgb-plggw.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:06 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-15-to-16 UnableToCreateStanzas command terminated with exit code 50: repo1-path = /pgbackrest/repo1 2026-02-20 19:22:06.720 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-02-20 19:22:06.733 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? postgrescluster-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:31 +0000 UTC Normal Job.batch major-upgrade-15-to-16-backup-vpgb Completed Job completed job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:52 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-gfw2 SuccessfulDelete delete Pod major-upgrade-15-to-16-instance1-gfw2-0 in StatefulSet major-upgrade-15-to-16-instance1-gfw2 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:52 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-jcr4 SuccessfulDelete delete Pod major-upgrade-15-to-16-instance1-jcr4-0 in StatefulSet major-upgrade-15-to-16-instance1-jcr4 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:53 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:53 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:53 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:53 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-588fd95b56 SuccessfulDelete Deleted pod: major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:53 +0000 UTC Normal Deployment.apps major-upgrade-15-to-16-pgbouncer ScalingReplicaSet Scaled down replica set major-upgrade-15-to-16-pgbouncer-588fd95b56 from 3 to 0 deployment-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:54 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:54 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-msjgj.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:54 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:54 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-588fd95b56 SuccessfulDelete Deleted pod: major-upgrade-15-to-16-pgbouncer-588fd95b56-nv86k replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:54 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-588fd95b56 SuccessfulDelete Deleted pod: major-upgrade-15-to-16-pgbouncer-588fd95b56-6btjb replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:57 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-pzj7 SuccessfulDelete delete Pod major-upgrade-15-to-16-instance1-pzj7-0 in StatefulSet major-upgrade-15-to-16-instance1-pzj7 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:22:59 +0000 UTC Warning Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:00 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-15-to-16-set-instance1 NoPods No matching pods found controllermanager logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:06 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:06 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:06 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-repo-host SuccessfulDelete delete Pod major-upgrade-15-to-16-repo-host-0 in StatefulSet major-upgrade-15-to-16-repo-host successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:11 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd Binding Scheduled Successfully assigned kuttl-test-equipped-lab/15-to-16-pgdata-9rhjd to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:11 +0000 UTC Normal Job.batch 15-to-16-pgdata SuccessfulCreate Created pod: 15-to-16-pgdata-9rhjd job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:15 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-28d721ca-b172-4140-87fb-5e5a8c0bc85b" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:23 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:23 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 238ms (238ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:23 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:23 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:23 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-relocator-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:24 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 258ms (258ms including waiting). Image size: 728720234 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:24 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:24 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:25 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:26 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 202ms (202ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:26 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:26 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:26 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-relocator-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:27 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-relocator-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 225ms (225ms including waiting). Image size: 728720234 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:27 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-relocator-16} Created Created container: extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:27 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-relocator-16} Started Started container extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:27 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-installer-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:28 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-installer-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 188ms (188ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:28 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-installer-16} Created Created container: extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:28 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.initContainers{extension-installer-16} Started Started container extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:29 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:29 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:30 +0000 UTC Normal Pod 15-to-16-pgdata-9rhjd.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:40 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz Binding Scheduled Successfully assigned kuttl-test-equipped-lab/15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:40 +0000 UTC Normal Job.batch 15-to-16-major-upgrade-15-to-16-instance1-gfw2 SuccessfulCreate Created pod: 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:40 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr Binding Scheduled Successfully assigned kuttl-test-equipped-lab/15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:40 +0000 UTC Normal Job.batch 15-to-16-major-upgrade-15-to-16-instance1-jcr4 SuccessfulCreate Created pod: 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:40 +0000 UTC Normal Job.batch 15-to-16-pgdata Completed Job completed job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:47 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-e6ab99f1-3a44-4a59-9ed0-17c17add560d" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:47 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-d0638d02-35e5-4b96-af8d-12af88b0fb04" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:49 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:49 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:49 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-gfw2-s9tgz.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:51 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:51 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:51 +0000 UTC Normal Pod 15-to-16-major-upgrade-15-to-16-instance1-jcr4-nk2dr.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:52 +0000 UTC Normal Job.batch 15-to-16-major-upgrade-15-to-16-instance1-gfw2 Completed Job completed job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:54 +0000 UTC Normal Job.batch 15-to-16-major-upgrade-15-to-16-instance1-jcr4 Completed Job completed job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:58 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-pzj7-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:58 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-65f99f5688 SuccessfulCreate Created pod: major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:58 +0000 UTC Normal Deployment.apps major-upgrade-15-to-16-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-15-to-16-pgbouncer-65f99f5688 from 0 to 3 deployment-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:58 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" in 221ms (221ms including waiting). Image size: 87593712 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-7g2j default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" in 243ms (243ms including waiting). Image size: 87593712 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" in 150ms (150ms including waiting). Image size: 87593712 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-65f99f5688 SuccessfulCreate Created pod: major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:23:59 +0000 UTC Normal ReplicaSet.apps major-upgrade-15-to-16-pgbouncer-65f99f5688 SuccessfulCreate Created pod: major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2 replicaset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" in 275ms (275ms including waiting). Image size: 87593712 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-g4dm2.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" in 232ms (233ms including waiting). Image size: 87593712 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-wzfcj.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer16" in 252ms (252ms including waiting). Image size: 87593712 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:00 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbouncer-65f99f5688-xbvzs.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:05 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-28d721ca-b172-4140-87fb-5e5a8c0bc85b" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:06 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:06 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 218ms (218ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:06 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:06 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-ff1c4cb0-8e5b-40c3-b2c7-28bc4e8b7861" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:07 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:07 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:07 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 246ms (246ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:07 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:08 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:08 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 240ms (240ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:08 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:08 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:08 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:08 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:08 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 223ms (223ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:09 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:09 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 200ms (200ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:09 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-16} Created Created container: extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:09 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-16} Started Started container extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:09 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:09 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:09 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:09 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 221ms (221ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 229ms (230ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-16} Created Created container: extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-16} Started Started container extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 252ms (252ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:11 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:11 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 231ms (231ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:11 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:11 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:12 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:12 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 250ms (250ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:12 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:12 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:12 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:17 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-gfw2-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-7g2j default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:18 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-jcr4-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:20 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-backup-dpvw-nzsk7 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:20 +0000 UTC Normal Job.batch major-upgrade-15-to-16-backup-dpvw SuccessfulCreate Created pod: major-upgrade-15-to-16-backup-dpvw-nzsk7 job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:21 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:21 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 202ms (202ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:21 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:21 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:22 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:22 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 192ms (192ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:22 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:22 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-dpvw-nzsk7.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:23 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-15-to-16 UnableToCreateStanzas command terminated with exit code 50: repo1-path = /pgbackrest/repo1 2026-02-20 19:24:23.069 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-archive-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-02-20 19:24:23.079 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-archive-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? postgrescluster-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:25 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-e6ab99f1-3a44-4a59-9ed0-17c17add560d" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:25 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-d0638d02-35e5-4b96-af8d-12af88b0fb04" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:26 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 258ms (258ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 261ms (261ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 250ms (250ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 258ms (258ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-16} Created Created container: extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-16} Started Started container extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 219ms (219ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 229ms (229ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-16} Created Created container: extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-16} Started Started container extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 239ms (239ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-16} Created Created container: extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-16} Started Started container extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 198ms (198ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-16} Created Created container: extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-16} Started Started container extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 269ms (269ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 219ms (219ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 231ms (231ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:34 +0000 UTC Normal Job.batch major-upgrade-15-to-16-backup-dpvw Completed Job completed job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 234ms (234ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:56 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:56 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:56 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-repo2 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:57 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-repo2 Provisioning External provisioner is provisioning volume for claim "kuttl-test-equipped-lab/major-upgrade-15-to-16-repo2" pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:57 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-repo2 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:24:57 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-15-to-16 UnableToCreateStanzas command terminated with exit code 1: postgres operator error: pgBackRest stale volume-backed repo configuration postgrescluster-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:01 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:01 +0000 UTC Normal PersistentVolumeClaim major-upgrade-15-to-16-repo2 ProvisioningSucceeded Successfully provisioned volume pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1 pd.csi.storage.gke.io_gke-d64f13f2694c4849a8d5-3cb0-2643-vm_4082c4c1-b2ef-42f8-8fea-9dc127fcb52e logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:09 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-505fe43f-1af8-40f2-8f2c-21ecf0184ec1" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 223ms (223ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:10 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:11 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:11 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 209ms (209ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:11 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:11 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:12 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:12 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 221ms (221ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:12 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:12 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:12 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:12 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 203ms (203ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:12 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:25:13 +0000 UTC Normal Pod major-upgrade-15-to-16-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:17 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-15-to-16 UnableToCreateStanzas command terminated with exit code 55: repo2-path = /pgbackrest/repo2 2026-02-20 19:26:16.999 P00 ERROR: [049]: timeout connecting to 'major-upgrade-15-to-16-repo-host-0.major-upgrade-15-to-16-pods.kuttl-test-equipped-lab.svc.cluster.local:8432 (10.62.41.43)' 2026-02-20 19:26:17.035 P00 ERROR: [055]: unable to load info file '/pgbackrest/repo2/archive/db/archive.info' or '/pgbackrest/repo2/archive/db/archive.info.copy': FileMissingError: raised from remote-0 tls protocol on 'major-upgrade-15-to-16-repo-host-0.major-upgrade-15-to-16-pods.kuttl-test-equipped-lab.svc.cluster.local': unable to open missing file '/pgbackrest/repo2/archive/db/archive.info' for read FileMissingError: raised from remote-0 tls protocol on 'major-upgrade-15-to-16-repo-host-0.major-upgrade-15-to-16-pods.kuttl-test-equipped-lab.svc.cluster.local': unable to open missing file '/pgbackrest/repo2/archive/db/archive.info.copy' for read HINT: archive.info cannot be opened but is required to push/get WAL segments. HINT: is archive_command configured correctly in postgresql.conf? HINT: has a stanza-create been performed? HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme. postgrescluster-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:29 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-backup-h8qp-bnljp to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:29 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:29 +0000 UTC Normal Job.batch major-upgrade-15-to-16-backup-h8qp SuccessfulCreate Created pod: major-upgrade-15-to-16-backup-h8qp-bnljp job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:30 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 252ms (252ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:30 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:30 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:31 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:31 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 201ms (201ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:31 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:31 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-h8qp-bnljp.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:44 +0000 UTC Normal Job.batch major-upgrade-15-to-16-backup-h8qp Completed Job completed job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:50 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-backup-6hhz-zw66b to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:50 +0000 UTC Normal Job.batch major-upgrade-15-to-16-backup-6hhz SuccessfulCreate Created pod: major-upgrade-15-to-16-backup-6hhz-zw66b job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:51 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:51 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 216ms (216ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:51 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:51 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:52 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:52 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 162ms (162ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:52 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:26:52 +0000 UTC Normal Pod major-upgrade-15-to-16-backup-6hhz-zw66b.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:02 +0000 UTC Normal Job.batch major-upgrade-15-to-16-backup-6hhz Completed Job completed job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:31 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-pgbackrest-restore-z74bv to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:31 +0000 UTC Normal Job.batch major-upgrade-15-to-16-pgbackrest-restore SuccessfulCreate Created pod: major-upgrade-15-to-16-pgbackrest-restore-z74bv job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:32 +0000 UTC Warning Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv FailedAttachVolume Multi-Attach error for volume "pvc-28d721ca-b172-4140-87fb-5e5a8c0bc85b" Volume is already exclusively attached to one node and can't be attached to another attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:51 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-28d721ca-b172-4140-87fb-5e5a8c0bc85b" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:53 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:53 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest16" in 215ms (215ms including waiting). Image size: 164325959 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:53 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:53 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:54 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv.spec.containers{pgbackrest-restore} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:54 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv.spec.containers{pgbackrest-restore} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 234ms (234ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:54 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv.spec.containers{pgbackrest-restore} Created Created container: pgbackrest-restore kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:27:54 +0000 UTC Normal Pod major-upgrade-15-to-16-pgbackrest-restore-z74bv.spec.containers{pgbackrest-restore} Started Started container pgbackrest-restore kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:04 +0000 UTC Normal Job.batch major-upgrade-15-to-16-pgbackrest-restore Completed Job completed job-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:05 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-pzj7-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-n7r3 default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:05 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-pzj7 SuccessfulCreate create Pod major-upgrade-15-to-16-instance1-pzj7-0 in StatefulSet major-upgrade-15-to-16-instance1-pzj7 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:06 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:06 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 216ms (216ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:06 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:07 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:07 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:07 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 236ms (236ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:07 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:07 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:08 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:08 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 248ms (248ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:08 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-16} Created Created container: extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:08 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-relocator-16} Started Started container extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:10 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:10 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 265ms (265ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:10 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-16} Created Created container: extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:10 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{extension-installer-16} Started Started container extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:11 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:11 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 239ms (239ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:11 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:11 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:12 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:12 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 259ms (259ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:12 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:13 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:13 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-pzj7-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:18 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-gfw2-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-7g2j default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:18 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-gfw2 SuccessfulCreate create Pod major-upgrade-15-to-16-instance1-gfw2-0 in StatefulSet major-upgrade-15-to-16-instance1-gfw2 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:18 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0 Binding Scheduled Successfully assigned kuttl-test-equipped-lab/major-upgrade-15-to-16-instance1-jcr4-0 to gke-jen-pg-1454-bcdf84ed-default-pool-cc470a6b-cm5h default-scheduler logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:18 +0000 UTC Normal StatefulSet.apps major-upgrade-15-to-16-instance1-jcr4 SuccessfulCreate create Pod major-upgrade-15-to-16-instance1-jcr4-0 in StatefulSet major-upgrade-15-to-16-instance1-jcr4 successful statefulset-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:22 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-e6ab99f1-3a44-4a59-9ed0-17c17add560d" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:26 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:26 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-d0638d02-35e5-4b96-af8d-12af88b0fb04" attachdetach-controller logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 259ms (259ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:27 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 255ms (255ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 160ms (160ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:28 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 239ms (239ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-16} Created Created container: extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:29 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-relocator-16} Started Started container extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 225ms (225ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:30 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 178ms (178ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-16} Created Created container: extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{extension-installer-16} Started Started container extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 219ms (219ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-16} Created Created container: extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:31 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-relocator-16} Started Started container extension-relocator-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 239ms (239ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:32 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-16} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 244ms (244ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-16} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 214ms (214ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-16} Created Created container: extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{extension-installer-16} Started Started container extension-installer-16 kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:33 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 217ms (217ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:34 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:35 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" in 175ms (175ms including waiting). Image size: 544763852 bytes. kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:35 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:35 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:35 +0000 UTC Normal Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg16-postgres" kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:37 +0000 UTC Warning Pod major-upgrade-15-to-16-instance1-gfw2-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | 2026-02-20 19:28:46 +0000 UTC Warning Pod major-upgrade-15-to-16-instance1-jcr4-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:52:31 | major-upgrade-15-to-16 | Deleting namespace "kuttl-test-equipped-lab" case.go:240: waiting for namespace "kuttl-test-equipped-lab" to be deleted timed out: failed to check deletion of namespace "kuttl-test-equipped-lab": context deadline exceeded === NAME kuttl harness.go:404: run tests finished harness.go:511: cleaning up harness.go:568: removing temp folder: "" --- FAIL: kuttl (2486.73s) --- FAIL: kuttl/harness (0.00s) --- FAIL: kuttl/harness/major-upgrade-15-to-16 (2486.01s) FAIL