=== RUN kuttl harness.go:460: starting setup harness.go:258: running tests using configured kubeconfig. harness.go:281: Successful connection to cluster at: https://35.194.35.254 harness.go:366: running tests harness.go:77: going to run test suite with timeout of 600 seconds for each step harness.go:378: testsuite: e2e-tests/tests has 36 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-13-to-14 === PAUSE kuttl/harness/major-upgrade-13-to-14 === CONT kuttl/harness/major-upgrade-13-to-14 logger.go:42: 19:19:08 | major-upgrade-13-to-14 | Creating namespace "kuttl-test-patient-primate" logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | + source ../../functions logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ realpath ../../.. logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ pwd logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-13-to-14 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++ test_name=major-upgrade-13-to-14 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ command -v oc logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ PG_VER=18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ printenv logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ which gdate logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ which date logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ which gsed logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++++ which sed logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | ++ oc get projects logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | + init_temp_dir logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | + deploy_operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | + local cw_prefix= logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | + destroy_operator logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 19:19:08 | major-upgrade-13-to-14/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 19:19:09 | major-upgrade-13-to-14/0-deploy-operator | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 19:19:09 | major-upgrade-13-to-14/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 19:19:09 | major-upgrade-13-to-14/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 logger.go:42: 19:19:09 | major-upgrade-13-to-14/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 19:19:10 | major-upgrade-13-to-14/0-deploy-operator | namespace "pg-operator" force deleted logger.go:42: 19:19:20 | major-upgrade-13-to-14/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 19:19:20 | major-upgrade-13-to-14/0-deploy-operator | + create_namespace pg-operator logger.go:42: 19:19:20 | major-upgrade-13-to-14/0-deploy-operator | + local namespace=pg-operator logger.go:42: 19:19:20 | major-upgrade-13-to-14/0-deploy-operator | + [[ -n '' ]] logger.go:42: 19:19:20 | major-upgrade-13-to-14/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 19:19:21 | major-upgrade-13-to-14/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 19:19:21 | major-upgrade-13-to-14/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 19:19:22 | major-upgrade-13-to-14/0-deploy-operator | namespace/pg-operator created logger.go:42: 19:19:22 | major-upgrade-13-to-14/0-deploy-operator | + cw_prefix=cw- logger.go:42: 19:19:22 | major-upgrade-13-to-14/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/crd.yaml logger.go:42: 19:19:23 | major-upgrade-13-to-14/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:19:23 | major-upgrade-13-to-14/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 19:19:25 | major-upgrade-13-to-14/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 19:19:26 | major-upgrade-13-to-14/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 19:19:26 | major-upgrade-13-to-14/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 19:19:27 | major-upgrade-13-to-14/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:19:27 | major-upgrade-13-to-14/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:19:28 | major-upgrade-13-to-14/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:19:28 | major-upgrade-13-to-14/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cw-rbac.yaml logger.go:42: 19:19:29 | major-upgrade-13-to-14/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 19:19:29 | major-upgrade-13-to-14/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 19:19:30 | major-upgrade-13-to-14/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 19:19:30 | major-upgrade-13-to-14/0-deploy-operator | + local disable_telemetry=true logger.go:42: 19:19:30 | major-upgrade-13-to-14/0-deploy-operator | + '[' major-upgrade-13-to-14 == telemetry-transfer ']' logger.go:42: 19:19:30 | major-upgrade-13-to-14/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cw-operator.yaml logger.go:42: 19:19:30 | major-upgrade-13-to-14/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 19:19:30 | major-upgrade-13-to-14/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 19:19:30 | major-upgrade-13-to-14/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 19:19:30 | major-upgrade-13-to-14/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 19:19:31 | major-upgrade-13-to-14/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 19:19:31 | major-upgrade-13-to-14/0-deploy-operator | + deploy_client logger.go:42: 19:19:31 | major-upgrade-13-to-14/0-deploy-operator | + kubectl -n kuttl-test-patient-primate apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf/client.yaml logger.go:42: 19:19:32 | major-upgrade-13-to-14/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 19:19:32 | major-upgrade-13-to-14/0-deploy-operator | + deploy_s3_secrets logger.go:42: 19:19:32 | major-upgrade-13-to-14/0-deploy-operator | + set +o xtrace logger.go:42: 19:19:33 | major-upgrade-13-to-14/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 19:19:34 | major-upgrade-13-to-14/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 19:19:34 | major-upgrade-13-to-14/0-deploy-operator | secret/azure-secret created logger.go:42: 19:19:35 | major-upgrade-13-to-14/0-deploy-operator | secret/minio-secret created logger.go:42: 19:19:36 | major-upgrade-13-to-14/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 19:19:36 | major-upgrade-13-to-14/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 19:19:37 | major-upgrade-13-to-14/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 19:19:37 | major-upgrade-13-to-14/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 19:19:37 | major-upgrade-13-to-14/0-deploy-operator | ASSERT PASS logger.go:42: 19:19:37 | major-upgrade-13-to-14/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | starting test step 1-create-cluster logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions get_cr \ | yq eval ' .spec.postgresVersion = 13 | .spec.image = "perconalab/percona-postgresql-operator:main-ppg13-postgres" | .spec.instances[0].dataVolumeClaimSpec.resources.requests.storage = "3Gi" | .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-pgbouncer13" | .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-pgbackrest13" | .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = "pg_cron" | .spec.extensions.custom += [{"name": "pg_cron", "version": "1.6.6"}]' \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + source ../../functions logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ realpath ../../.. logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ pwd logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-13-to-14 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++ test_name=major-upgrade-13-to-14 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ command -v oc logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export PG_VER=18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ PG_VER=18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ printenv logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ which gdate logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ which date logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ which gsed logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++++ which sed logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ++ oc get projects logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + get_cr logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + local cr_name= logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + '[' -z ']' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + cr_name=major-upgrade-13-to-14 logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + local repo_path= logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + local source_path= logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + yq eval ' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .metadata.name = "major-upgrade-13-to-14" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .metadata.labels = {"e2e":"major-upgrade-13-to-14"} | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.postgresVersion = 18 | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.users += [{"name":"major-upgrade-13-to-14","password":{"type":"AlphaNumeric"}}] | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.pmm.image = "docker.io/perconalab/pmm-client:dev-latest" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.pmm.secret = "major-upgrade-13-to-14-pmm-secret" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.pmm.customClusterName = "major-upgrade-13-to-14-pmm-custom-name" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.pmm.postgresParams = "--environment=dev-postgres" logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ' /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cr.yaml logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + yq eval ' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.postgresVersion = 13 | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.image = "perconalab/percona-postgresql-operator:main-ppg13-postgres" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.instances[0].dataVolumeClaimSpec.resources.requests.storage = "3Gi" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-pgbouncer13" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-pgbackrest13" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = "pg_cron" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.extensions.custom += [{"name": "pg_cron", "version": "1.6.6"}]' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + kubectl -n kuttl-test-patient-primate apply -f - logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + [[ -n '' ]] logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + case $test_name in logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + yq eval -i ' logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.extensions.image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.extensions.imagePullPolicy = "Always" | logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | .spec.extensions.storage = {"type": "s3", "bucket": "pg-extensions", "region": "eu-central-1", "secret": {"name": "aws-s3-secret"}} logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | ' /tmp/kuttl/pg/major-upgrade-13-to-14/cr.yaml logger.go:42: 19:19:37 | major-upgrade-13-to-14/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-13-to-14/cr.yaml logger.go:42: 19:19:38 | major-upgrade-13-to-14/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 created logger.go:42: 19:20:40 | major-upgrade-13-to-14/1-create-cluster | test step completed 1-create-cluster logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | starting test step 2-write-data logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:$(get_psql_user_pass major-upgrade-13-to-14-pguser-postgres)@$(get_psql_user_host major-upgrade-13-to-14-pguser-postgres)" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:$(get_psql_user_pass major-upgrade-13-to-14-pguser-postgres)@$(get_psql_user_host major-upgrade-13-to-14-pguser-postgres)" run_psql_local \ '\c postgres \\\ CREATE EXTENSION pg_cron' \ "postgres:$(get_psql_user_pass major-upgrade-13-to-14-pguser-postgres)@$(get_psql_user_host major-upgrade-13-to-14-pguser-postgres)"] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | + source ../../functions logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ realpath ../../.. logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ pwd logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-13-to-14 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++ test_name=major-upgrade-13-to-14 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ command -v oc logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ -n '' ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export PG_VER=18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ PG_VER=18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ PGOV1_VER=14 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ printenv logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ which gdate logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ which date logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ date=/usr/sbin/date logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ which gsed logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++++ which sed logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++ oc get projects logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++ get_psql_user_pass major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:40 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | ++ get_psql_user_host major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | + local uri=postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | + local driver=postgres logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | ++ get_client_pod logger.go:42: 19:20:41 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:20:42 | major-upgrade-13-to-14/2-write-data | + kubectl -n kuttl-test-patient-primate exec pg-client-65d98588cc-cc9lq -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc'\''' logger.go:42: 19:20:43 | major-upgrade-13-to-14/2-write-data | ++ get_psql_user_pass major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:43 | major-upgrade-13-to-14/2-write-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:43 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:20:43 | major-upgrade-13-to-14/2-write-data | ++ get_psql_user_host major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:43 | major-upgrade-13-to-14/2-write-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:43 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:20:44 | major-upgrade-13-to-14/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:20:44 | major-upgrade-13-to-14/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 19:20:44 | major-upgrade-13-to-14/2-write-data | + local uri=postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:20:44 | major-upgrade-13-to-14/2-write-data | + local driver=postgres logger.go:42: 19:20:44 | major-upgrade-13-to-14/2-write-data | ++ get_client_pod logger.go:42: 19:20:44 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:20:44 | major-upgrade-13-to-14/2-write-data | + kubectl -n kuttl-test-patient-primate exec pg-client-65d98588cc-cc9lq -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc'\''' logger.go:42: 19:20:45 | major-upgrade-13-to-14/2-write-data | ++ get_psql_user_pass major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:45 | major-upgrade-13-to-14/2-write-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:45 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | ++ get_psql_user_host major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | + run_psql_local '\c postgres \\\ CREATE EXTENSION pg_cron' postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | + local 'command=\c postgres \\\ CREATE EXTENSION pg_cron' logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | + local uri=postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | + local driver=postgres logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | ++ get_client_pod logger.go:42: 19:20:46 | major-upgrade-13-to-14/2-write-data | ++ kubectl -n kuttl-test-patient-primate get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:20:47 | major-upgrade-13-to-14/2-write-data | + kubectl -n kuttl-test-patient-primate exec pg-client-65d98588cc-cc9lq -- bash -c 'printf '\''\c postgres \\\ CREATE EXTENSION pg_cron\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc'\''' logger.go:42: 19:20:48 | major-upgrade-13-to-14/2-write-data | test step completed 2-write-data logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | starting test step 3-read-from-primary logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-13-to-14-pguser-postgres)@$(get_psql_user_host major-upgrade-13-to-14-pguser-postgres)") kubectl create configmap -n "${NAMESPACE}" 03-read-from-primary --from-literal=data="${data}"] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | + source ../../functions logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ realpath ../../.. logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ pwd logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-13-to-14 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++ test_name=major-upgrade-13-to-14 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ command -v oc logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ ! -n '' ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ -n '' ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export PG_VER=18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ PG_VER=18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export BUCKET=pg-operator-testing logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ BUCKET=pg-operator-testing logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export PGOV1_VER=14 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ PGOV1_VER=14 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ MINIO_VER=5.4.0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ REGISTRY_NAME=docker.io logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ printenv logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ grep -E '^IMAGE' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ awk -F= '{print $1}' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ which gdate logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ which date logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ date=/usr/sbin/date logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ which gsed logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++++ which sed logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ sed=/usr/sbin/sed logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | ++ oc get projects logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ get_psql_user_pass major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:48 | major-upgrade-13-to-14/3-read-from-primary | +++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | +++ get_psql_user_host major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | +++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | +++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | ++ run_psql_local '\c myapp \\\ SELECT * from myApp;' postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | ++ local 'command=\c myapp \\\ SELECT * from myApp;' logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | ++ local uri=postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | ++ local driver=postgres logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | +++ get_client_pod logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | +++ kubectl -n kuttl-test-patient-primate get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:20:49 | major-upgrade-13-to-14/3-read-from-primary | ++ kubectl -n kuttl-test-patient-primate exec pg-client-65d98588cc-cc9lq -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc'\''' logger.go:42: 19:20:51 | major-upgrade-13-to-14/3-read-from-primary | + data=' 100500' logger.go:42: 19:20:51 | major-upgrade-13-to-14/3-read-from-primary | + kubectl create configmap -n kuttl-test-patient-primate 03-read-from-primary '--from-literal=data= 100500' logger.go:42: 19:20:51 | major-upgrade-13-to-14/3-read-from-primary | configmap/03-read-from-primary created logger.go:42: 19:20:52 | major-upgrade-13-to-14/3-read-from-primary | test step completed 3-read-from-primary logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | starting test step 4-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c set -o errexit set -o xtrace source ../../functions kubectl apply -n ${NAMESPACE} -f - <<-EOF apiVersion: pgv2.percona.com/v2 kind: PerconaPGUpgrade metadata: name: 13-to-14 spec: postgresClusterName: major-upgrade-13-to-14 image: ${IMAGE_UPGRADE} fromPostgresVersion: 13 toPostgresVersion: 14 toPostgresImage: $(get_container_image "postgres" 14) toPgBouncerImage: $(get_container_image "pgbouncer" 14) toPgBackRestImage: $(get_container_image "pgbackrest" 14) EOF] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | + source ../../functions logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ realpath ../../.. logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ pwd logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-13-to-14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ test_name=major-upgrade-13-to-14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ command -v oc logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ ! -n '' ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ -n '' ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export PG_VER=18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ PG_VER=18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export BUCKET=pg-operator-testing logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ BUCKET=pg-operator-testing logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export PGOV1_VER=14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ PGOV1_VER=14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export MINIO_VER=5.4.0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ MINIO_VER=5.4.0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ REGISTRY_NAME=docker.io logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ printenv logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ grep -E '^IMAGE' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ awk -F= '{print $1}' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ which gdate logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ which date logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ date=/usr/sbin/date logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ which gsed logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++++ which sed logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | +++ sed=/usr/sbin/sed logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ oc get projects logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | + kubectl apply -n kuttl-test-patient-primate -f - logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ get_container_image postgres 14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local component=postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local pgVersion=14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ operatorVersion=main logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ [[ postgres == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ [[ postgres == \p\g\b\a\c\k\r\e\s\t ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ get_container_image pgbouncer 14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local component=pgbouncer logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local pgVersion=14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ operatorVersion=main logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ [[ pgbouncer == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ get_container_image pgbackrest 14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local component=pgbackrest logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local pgVersion=14 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ operatorVersion=main logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ [[ pgbackrest == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ [[ pgbackrest == \p\g\b\a\c\k\r\e\s\t ]] logger.go:42: 19:20:52 | major-upgrade-13-to-14/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14 logger.go:42: 19:20:53 | major-upgrade-13-to-14/4-upgrade | perconapgupgrade.pgv2.percona.com/13-to-14 created logger.go:42: 19:20:54 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc ready 3 3 76s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 48s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 75s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-dr44-0 4/4 Running 0 75s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-zwr5-0 4/4 Running 0 75s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr 2/2 Running 0 74s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc 2/2 Running 0 73s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7 2/2 Running 0 73s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 2/2 Running 0 74s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 82s logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:20:55 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 49s logger.go:42: 19:21:01 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc stopping 3 3 83s logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 56s logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 83s logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 2/2 Running 0 82s logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 90s logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:02 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 56s logger.go:42: 19:21:08 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc paused 0 0 91s logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Init:0/5 0 1s logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 63s logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 0/2 Completed 0 89s logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 97s logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Running 0/1 1s 1s logger.go:42: 19:21:09 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 63s logger.go:42: 19:21:16 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc paused 0 0 98s logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Init:0/5 0 8s logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 70s logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 104s logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Running 0/1 8s 8s logger.go:42: 19:21:17 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 70s logger.go:42: 19:21:23 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc paused 0 0 105s logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Init:0/5 0 15s logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 77s logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 111s logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Running 0/1 16s 16s logger.go:42: 19:21:24 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 78s logger.go:42: 19:21:30 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc paused 0 0 112s logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Init:1/5 0 22s logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 84s logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 118s logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Running 0/1 23s 23s logger.go:42: 19:21:31 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 85s logger.go:42: 19:21:37 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc paused 0 0 119s logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 1/1 Running 0 30s logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 92s logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 2m6s logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Running 0/1 30s 30s logger.go:42: 19:21:38 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 92s logger.go:42: 19:21:44 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc paused 0 0 2m6s logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 37s logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 99s logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 2m13s logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Running 0/1 37s 37s logger.go:42: 19:21:45 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 99s logger.go:42: 19:21:51 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc paused 0 0 2m14s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 ContainerCreating 0 5s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 ContainerCreating 0 5s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 44s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 106s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 2m20s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Running 0/1 5s 6s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Running 0/1 5s 5s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 44s logger.go:42: 19:21:52 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 106s logger.go:42: 19:21:58 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc paused 0 0 2m21s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 12s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 ContainerCreating 0 12s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 51s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-qsgq-lnnrz 0/1 Completed 0 113s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 2m27s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 13s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Running 0/1 12s 12s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 51s logger.go:42: 19:21:59 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-qsgq Complete 1/1 32s 113s logger.go:42: 19:22:06 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc initializing 0 0 2m28s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 19s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 19s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 58s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 0/4 Init:0/5 0 1s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 0/2 Pending 0 1s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 0/2 ContainerCreating 0 1s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 0/2 ContainerCreating 0 1s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 0/2 Init:0/2 0 1s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 2m34s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 21s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Complete 1/1 14s 20s logger.go:42: 19:22:07 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 59s logger.go:42: 19:22:13 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc initializing 0 3 2m35s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 27s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 27s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 66s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 0/4 Init:0/5 0 9s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 9s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 9s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 9s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 0/2 Init:0/2 0 9s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 2m42s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 28s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Complete 1/1 14s 27s logger.go:42: 19:22:14 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 66s logger.go:42: 19:22:20 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc initializing 0 3 2m43s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 34s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 34s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 73s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 0/4 Init:3/5 0 16s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 16s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 16s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 16s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 2/2 Running 0 16s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 2m49s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 35s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Complete 1/1 14s 34s logger.go:42: 19:22:21 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 73s logger.go:42: 19:22:27 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc initializing 0 3 2m50s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 41s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 41s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 80s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 23s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-zwr5-0 0/4 Pending 0 0s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 23s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 23s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 23s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 2/2 Running 0 23s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 2m56s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 42s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Complete 1/1 14s 41s logger.go:42: 19:22:28 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 80s logger.go:42: 19:22:35 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc initializing 1 3 2m57s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 48s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 48s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 87s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-6zdd-dcj2t 1/1 Running 0 4s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 30s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-dr44-0 0/4 Init:0/5 0 7s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-zwr5-0 0/4 Init:0/5 0 7s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 30s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 30s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 30s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 2/2 Running 0 30s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 3m3s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 50s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Complete 1/1 14s 49s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 88s logger.go:42: 19:22:36 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-6zdd Running 0/1 5s 5s logger.go:42: 19:22:42 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc initializing 1 3 3m5s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 56s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 56s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 95s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-6zdd-dcj2t 1/1 Running 0 12s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 38s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-dr44-0 0/4 Init:3/5 0 15s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-zwr5-0 0/4 PodInitializing 0 15s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 38s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 38s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 38s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 2/2 Running 0 38s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 3m11s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 57s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Complete 1/1 14s 56s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 95s logger.go:42: 19:22:43 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-6zdd Running 0/1 12s 12s logger.go:42: 19:22:50 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc initializing 2 3 3m12s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 63s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 63s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 102s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-6zdd-dcj2t 0/1 Completed 0 19s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 45s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-dr44-0 4/4 Running 0 22s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-zwr5-0 4/4 Running 0 22s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 45s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 45s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 45s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 2/2 Running 0 45s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 3m18s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 65s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Complete 1/1 14s 64s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 103s logger.go:42: 19:22:51 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-6zdd Complete 1/1 15s 20s logger.go:42: 19:22:57 | major-upgrade-13-to-14/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc ready 3 3 3m19s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 70s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 70s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/13-to-14-pgdata-klq9k 0/1 Completed 0 109s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-backup-6zdd-dcj2t 0/1 Completed 0 26s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 52s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-dr44-0 4/4 Running 0 29s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-instance1-zwr5-0 4/4 Running 0 29s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 52s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 52s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 52s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/major-upgrade-13-to-14-repo-host-0 2/2 Running 0 52s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | pod/pg-client-65d98588cc-cc9lq 1/1 Running 0 3m25s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-dr44 Complete 1/1 11s 72s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-major-upgrade-13-to-14-instance1-zwr5 Complete 1/1 14s 71s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | job.batch/13-to-14-pgdata Complete 1/1 38s 110s logger.go:42: 19:22:58 | major-upgrade-13-to-14/4-upgrade | job.batch/major-upgrade-13-to-14-backup-6zdd Complete 1/1 15s 27s logger.go:42: 19:23:03 | major-upgrade-13-to-14/4-upgrade | test step completed 4-upgrade logger.go:42: 19:23:03 | major-upgrade-13-to-14/5-change-repo | starting test step 5-change-repo logger.go:42: 19:23:03 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c set -o errexit set -o xtrace kubectl patch \ -n $NAMESPACE \ perconapgcluster major-upgrade-13-to-14 \ --type='json' \ -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]'] logger.go:42: 19:23:03 | major-upgrade-13-to-14/5-change-repo | + kubectl patch -n kuttl-test-patient-primate perconapgcluster major-upgrade-13-to-14 --type=json '-p=[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]' logger.go:42: 19:23:04 | major-upgrade-13-to-14/5-change-repo | perconapgcluster.pgv2.percona.com/major-upgrade-13-to-14 patched logger.go:42: 19:23:04 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:05 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:05 | major-upgrade-13-to-14/5-change-repo | name: repo1 logger.go:42: 19:23:05 | major-upgrade-13-to-14/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:23:05 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: true logger.go:42: 19:23:05 | major-upgrade-13-to-14/5-change-repo | volume: pvc-c439ce78-9e23-42c0-b844-7a5acc3779a2 logger.go:42: 19:23:06 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:07 | major-upgrade-13-to-14/5-change-repo | - name: repo2 logger.go:42: 19:23:07 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:08 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:08 | major-upgrade-13-to-14/5-change-repo | - name: repo2 logger.go:42: 19:23:08 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:09 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:10 | major-upgrade-13-to-14/5-change-repo | - name: repo2 logger.go:42: 19:23:10 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:11 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:12 | major-upgrade-13-to-14/5-change-repo | - name: repo2 logger.go:42: 19:23:12 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:13 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:14 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:14 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:14 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:14 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:15 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:15 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:15 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:15 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:15 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:17 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:17 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:17 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:17 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:17 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:18 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:19 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:19 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:19 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:19 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:20 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:21 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:21 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:21 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:21 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:22 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:22 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:22 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:22 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:22 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:24 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:24 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:24 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:24 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:24 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:25 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:26 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:26 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:26 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:26 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:27 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:28 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:28 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:28 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:28 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:29 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:29 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:29 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:29 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:29 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:31 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:31 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:31 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:31 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:31 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:32 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:33 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:33 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:33 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:33 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:34 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:35 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:35 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:35 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:35 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:36 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:36 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:36 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:36 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:36 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:38 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:38 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:38 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:38 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: false logger.go:42: 19:23:38 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:39 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:40 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:40 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:40 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: true logger.go:42: 19:23:40 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:41 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:42 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:42 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:42 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: true logger.go:42: 19:23:42 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:43 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:44 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:44 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:44 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: true logger.go:42: 19:23:44 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:45 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:45 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:45 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:45 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: true logger.go:42: 19:23:45 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:46 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:47 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:47 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:47 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: true logger.go:42: 19:23:47 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:48 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:49 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:49 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:49 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: true logger.go:42: 19:23:49 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:50 | major-upgrade-13-to-14/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-13-to-14 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:23:51 | major-upgrade-13-to-14/5-change-repo | - bound: true logger.go:42: 19:23:51 | major-upgrade-13-to-14/5-change-repo | name: repo2 logger.go:42: 19:23:51 | major-upgrade-13-to-14/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:23:51 | major-upgrade-13-to-14/5-change-repo | stanzaCreated: true logger.go:42: 19:23:51 | major-upgrade-13-to-14/5-change-repo | volume: pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 logger.go:42: 19:23:51 | major-upgrade-13-to-14/5-change-repo | test step completed 5-change-repo logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | starting test step 6-write-data logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' \ "postgres:$(get_psql_user_pass major-upgrade-13-to-14-pguser-postgres)@$(get_psql_user_host major-upgrade-13-to-14-pguser-postgres)"] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | + source ../../functions logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ realpath ../../.. logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ pwd logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-13-to-14 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ test_name=major-upgrade-13-to-14 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ command -v oc logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ ! -n '' ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ -n '' ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export PG_VER=18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ PG_VER=18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export PGOV1_VER=14 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ PGOV1_VER=14 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ printenv logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ grep -E '^IMAGE' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ awk -F= '{print $1}' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ which gdate logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ which date logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ date=/usr/sbin/date logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ which gsed logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++++ which sed logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | +++ sed=/usr/sbin/sed logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ oc get projects logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ get_psql_user_pass major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ get_psql_user_host major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:23:51 | major-upgrade-13-to-14/6-write-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:23:52 | major-upgrade-13-to-14/6-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:23:52 | major-upgrade-13-to-14/6-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' logger.go:42: 19:23:52 | major-upgrade-13-to-14/6-write-data | + local uri=postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:23:52 | major-upgrade-13-to-14/6-write-data | + local driver=postgres logger.go:42: 19:23:52 | major-upgrade-13-to-14/6-write-data | ++ get_client_pod logger.go:42: 19:23:52 | major-upgrade-13-to-14/6-write-data | ++ kubectl -n kuttl-test-patient-primate get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:23:52 | major-upgrade-13-to-14/6-write-data | + kubectl -n kuttl-test-patient-primate exec pg-client-65d98588cc-cc9lq -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc'\''' logger.go:42: 19:23:53 | major-upgrade-13-to-14/6-write-data | test step completed 6-write-data logger.go:42: 19:23:53 | major-upgrade-13-to-14/7-run-backup | starting test step 7-run-backup logger.go:42: 19:23:54 | major-upgrade-13-to-14/7-run-backup | PerconaPGBackup:kuttl-test-patient-primate/backup-after-13-to-14 created logger.go:42: 19:24:23 | major-upgrade-13-to-14/7-run-backup | test step completed 7-run-backup logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | starting test step 8-read-from-primary logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-13-to-14-pguser-postgres)@$(get_psql_user_host major-upgrade-13-to-14-pguser-postgres)") kubectl create configmap -n "${NAMESPACE}" 08-read-from-primary --from-literal=data="${data}"] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | + source ../../functions logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ realpath ../../.. logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ pwd logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-13-to-14 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++ test_name=major-upgrade-13-to-14 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ command -v oc logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ ! -n '' ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ -n '' ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export PG_VER=18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ PG_VER=18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export BUCKET=pg-operator-testing logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ BUCKET=pg-operator-testing logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export PGOV1_VER=14 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ PGOV1_VER=14 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ MINIO_VER=5.4.0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ REGISTRY_NAME=docker.io logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ printenv logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ grep -E '^IMAGE' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ awk -F= '{print $1}' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ which gdate logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ which date logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ date=/usr/sbin/date logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ which gsed logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++++ which sed logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ sed=/usr/sbin/sed logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | ++ oc get projects logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ get_psql_user_pass major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ get_psql_user_host major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:24:23 | major-upgrade-13-to-14/8-read-from-primary | +++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:24:24 | major-upgrade-13-to-14/8-read-from-primary | ++ run_psql_local '\c myapp \\\ SELECT * from myApp;' postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:24:24 | major-upgrade-13-to-14/8-read-from-primary | ++ local 'command=\c myapp \\\ SELECT * from myApp;' logger.go:42: 19:24:24 | major-upgrade-13-to-14/8-read-from-primary | ++ local uri=postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:24:24 | major-upgrade-13-to-14/8-read-from-primary | ++ local driver=postgres logger.go:42: 19:24:24 | major-upgrade-13-to-14/8-read-from-primary | +++ get_client_pod logger.go:42: 19:24:24 | major-upgrade-13-to-14/8-read-from-primary | +++ kubectl -n kuttl-test-patient-primate get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:24:24 | major-upgrade-13-to-14/8-read-from-primary | ++ kubectl -n kuttl-test-patient-primate exec pg-client-65d98588cc-cc9lq -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc'\''' logger.go:42: 19:24:25 | major-upgrade-13-to-14/8-read-from-primary | + data=' 100500 logger.go:42: 19:24:25 | major-upgrade-13-to-14/8-read-from-primary | 100501' logger.go:42: 19:24:25 | major-upgrade-13-to-14/8-read-from-primary | + kubectl create configmap -n kuttl-test-patient-primate 08-read-from-primary '--from-literal=data= 100500 logger.go:42: 19:24:25 | major-upgrade-13-to-14/8-read-from-primary | 100501' logger.go:42: 19:24:26 | major-upgrade-13-to-14/8-read-from-primary | configmap/08-read-from-primary created logger.go:42: 19:24:26 | major-upgrade-13-to-14/8-read-from-primary | test step completed 8-read-from-primary logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | starting test step 9-delete-data logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ '\c myapp \\\ TRUNCATE TABLE myApp' \ "postgres:$(get_psql_user_pass major-upgrade-13-to-14-pguser-postgres)@$(get_psql_user_host major-upgrade-13-to-14-pguser-postgres)"] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | + source ../../functions logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ realpath ../../.. logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ pwd logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-13-to-14 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++ test_name=major-upgrade-13-to-14 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-13-to-14 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ command -v oc logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ ! -n '' ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ -n '' ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export PG_VER=18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ PG_VER=18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export PGOV1_VER=14 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ PGOV1_VER=14 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ printenv logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ grep -E '^IMAGE' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ awk -F= '{print $1}' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ which gdate logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ which date logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ date=/usr/sbin/date logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ which gsed logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++++ which sed logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | +++ sed=/usr/sbin/sed logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++ oc get projects logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++ get_psql_user_pass major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:24:26 | major-upgrade-13-to-14/9-delete-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | ++ get_psql_user_host major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | ++ local secret_name=major-upgrade-13-to-14-pguser-postgres logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | ++ kubectl -n kuttl-test-patient-primate get secret/major-upgrade-13-to-14-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | + run_psql_local '\c myapp \\\ TRUNCATE TABLE myApp' postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | + local 'command=\c myapp \\\ TRUNCATE TABLE myApp' logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | + local uri=postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | + local driver=postgres logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | ++ get_client_pod logger.go:42: 19:24:27 | major-upgrade-13-to-14/9-delete-data | ++ kubectl -n kuttl-test-patient-primate get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:24:28 | major-upgrade-13-to-14/9-delete-data | + kubectl -n kuttl-test-patient-primate exec pg-client-65d98588cc-cc9lq -- bash -c 'printf '\''\c myapp \\\ TRUNCATE TABLE myApp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:QfbkjTC0rgzlxzg0PJpdBRI8@major-upgrade-13-to-14-primary.kuttl-test-patient-primate.svc'\''' logger.go:42: 19:24:29 | major-upgrade-13-to-14/9-delete-data | test step completed 9-delete-data logger.go:42: 19:24:29 | major-upgrade-13-to-14/10-run-restore | starting test step 10-run-restore logger.go:42: 19:24:29 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-13-to-14 primary name) latest_full_repo2_backup=$(kubectl -n ${NAMESPACE} exec ${primary} -- pgbackrest info --output json --log-level-console=info | jq -r '[.[] | .backup[] | select(.type == "full") | select(.database["repo-key"] == 2)][-1].label') cat </dev/null done sleep 30] logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 2m46s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 2m46s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 3m25s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 38s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 55s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 2m28s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 4/4 Running 0 2m5s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 4/4 Running 0 2m5s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 2m28s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 2m28s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 2m28s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 87s logger.go:42: 19:24:34 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 5m1s logger.go:42: 19:24:35 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:24:38 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 2m51s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 2m51s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 3m30s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 43s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 60s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbackrest-restore-q26hz 0/1 Init:0/1 0 1s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 2m33s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 2m33s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 2m33s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 92s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 5m6s logger.go:42: 19:24:39 | major-upgrade-13-to-14/10-run-restore | No resources found in kuttl-test-patient-primate namespace. logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 3m24s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 3m24s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 4m3s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 76s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 93s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 0/4 Init:2/5 0 6s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbackrest-restore-q26hz 0/1 Completed 0 34s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 3m6s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 3m6s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 3m6s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 2m5s logger.go:42: 19:25:11 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 5m39s logger.go:42: 19:25:13 | major-upgrade-13-to-14/10-run-restore | Waiting for major-upgrade-13-to-14-instance1-6vhh-0 to start running logger.go:42: 19:25:44 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 3m57s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 3m57s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 4m36s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 109s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 2m6s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 39s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 23s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 24s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 3m39s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 3m39s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 3m39s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 2m38s logger.go:42: 19:25:45 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 6m12s logger.go:42: 19:25:46 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:25:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:25:48 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.884 UTC [129] LOG: received fast shutdown request logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.887 UTC [129] LOG: aborting any active transactions logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.887 UTC [160] FATAL: terminating connection due to administrator command logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.887 UTC [187] FATAL: terminating connection due to administrator command logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.889 UTC [154] FATAL: terminating walreceiver process due to administrator command logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.891 UTC [148] LOG: shutting down logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.914 UTC [129] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.961 UTC [129] LOG: database system is shut down logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:37.797 UTC [141] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:37.798 UTC [141] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:37.798 UTC [141] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:37.802 UTC [141] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:37.818 UTC [144] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:37.843 UTC [144] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:37.869 UTC [144] LOG: entering standby mode logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:37.895 UTC [144] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.224 UTC [144] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.255 UTC [144] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.256 UTC [141] LOG: startup process (PID 144) exited with exit code 1 logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.257 UTC [141] LOG: aborting startup due to startup process failure logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.257 UTC [141] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.303 UTC [141] LOG: database system is shut down logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.610 UTC [191] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.611 UTC [191] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.611 UTC [191] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.637 UTC [191] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.645 UTC [194] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.685 UTC [194] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.719 UTC [194] LOG: entering standby mode logger.go:42: 19:25:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.748 UTC [194] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:50 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:34.966 UTC [128] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:24:35.014 UTC [128] LOG: database system is shut down logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.172 UTC [141] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.173 UTC [141] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.173 UTC [141] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.199 UTC [141] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.217 UTC [144] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.249 UTC [144] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.282 UTC [144] LOG: entering standby mode logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.314 UTC [144] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.648 UTC [144] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.681 UTC [144] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.683 UTC [141] LOG: startup process (PID 144) exited with exit code 1 logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.683 UTC [141] LOG: aborting startup due to startup process failure logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.683 UTC [141] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:38.732 UTC [141] LOG: database system is shut down logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.739 UTC [193] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.740 UTC [193] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.740 UTC [193] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.766 UTC [193] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.775 UTC [196] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.819 UTC [196] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.855 UTC [196] LOG: entering standby mode logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:49.890 UTC [196] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:50.220 UTC [196] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:50.255 UTC [196] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:50.257 UTC [193] LOG: startup process (PID 196) exited with exit code 1 logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:50.257 UTC [193] LOG: aborting startup due to startup process failure logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:50.257 UTC [193] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:50.305 UTC [193] LOG: database system is shut down logger.go:42: 19:26:23 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 4m36s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 4m36s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 5m15s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 2m28s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 2m45s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 78s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 62s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 63s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 4m18s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 4m18s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 4m18s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 3m17s logger.go:42: 19:26:24 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 6m51s logger.go:42: 19:26:25 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:26:27 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:50.107 UTC [191] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:50.154 UTC [191] LOG: database system is shut down logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:09.599 UTC [265] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:09.600 UTC [265] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:09.600 UTC [265] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:09.625 UTC [265] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:09.635 UTC [268] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:09.678 UTC [268] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:09.711 UTC [268] LOG: entering standby mode logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:09.743 UTC [268] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:10.066 UTC [268] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:10.100 UTC [268] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:10.102 UTC [265] LOG: startup process (PID 268) exited with exit code 1 logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:10.102 UTC [265] LOG: aborting startup due to startup process failure logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:10.102 UTC [265] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:10.149 UTC [265] LOG: database system is shut down logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.631 UTC [307] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.631 UTC [307] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.631 UTC [307] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.665 UTC [307] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.676 UTC [316] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.707 UTC [316] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.738 UTC [316] LOG: entering standby mode logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.766 UTC [316] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.888 UTC [316] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.923 UTC [316] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.925 UTC [307] LOG: startup process (PID 316) exited with exit code 1 logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.925 UTC [307] LOG: aborting startup due to startup process failure logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.925 UTC [307] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.977 UTC [307] LOG: database system is shut down logger.go:42: 19:26:29 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:10.138 UTC [267] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:10.186 UTC [267] LOG: database system is shut down logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.645 UTC [309] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.646 UTC [309] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.646 UTC [309] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.671 UTC [309] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.679 UTC [318] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.714 UTC [318] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.742 UTC [318] LOG: entering standby mode logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.770 UTC [318] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:20.105 UTC [318] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:20.140 UTC [318] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:20.142 UTC [309] LOG: startup process (PID 318) exited with exit code 1 logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:20.142 UTC [309] LOG: aborting startup due to startup process failure logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:20.142 UTC [309] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:20.191 UTC [309] LOG: database system is shut down logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:29.638 UTC [358] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:29.639 UTC [358] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:29.639 UTC [358] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:29.664 UTC [358] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:29.672 UTC [361] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:29.701 UTC [361] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:29.730 UTC [361] LOG: entering standby mode logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:29.757 UTC [361] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:30.080 UTC [361] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:30.115 UTC [361] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:30.116 UTC [358] LOG: startup process (PID 361) exited with exit code 1 logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:30.116 UTC [358] LOG: aborting startup due to startup process failure logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:30.116 UTC [358] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:30.165 UTC [358] LOG: database system is shut down logger.go:42: 19:27:02 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 5m15s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 5m15s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 5m54s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 3m7s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 3m24s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 117s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 101s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 102s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 4m57s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 4m57s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 4m57s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 3m56s logger.go:42: 19:27:03 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 7m30s logger.go:42: 19:27:04 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:27:06 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.925 UTC [307] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:19.977 UTC [307] LOG: database system is shut down logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:39.623 UTC [381] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:39.623 UTC [381] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:39.623 UTC [381] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:39.649 UTC [381] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:39.659 UTC [390] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:39.696 UTC [390] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:39.726 UTC [390] LOG: entering standby mode logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:39.753 UTC [390] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:40.075 UTC [390] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:40.110 UTC [390] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:40.111 UTC [381] LOG: startup process (PID 390) exited with exit code 1 logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:40.111 UTC [381] LOG: aborting startup due to startup process failure logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:40.111 UTC [381] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:40.160 UTC [381] LOG: database system is shut down logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.639 UTC [447] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.640 UTC [447] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.640 UTC [447] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.666 UTC [447] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.674 UTC [459] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.700 UTC [459] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.731 UTC [459] LOG: entering standby mode logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.758 UTC [459] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.081 UTC [459] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.116 UTC [459] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.117 UTC [447] LOG: startup process (PID 459) exited with exit code 1 logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.117 UTC [447] LOG: aborting startup due to startup process failure logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.117 UTC [447] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.164 UTC [447] LOG: database system is shut down logger.go:42: 19:27:08 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:49.731 UTC [435] LOG: entering standby mode logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:49.760 UTC [435] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:50.083 UTC [435] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:50.119 UTC [435] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:50.120 UTC [432] LOG: startup process (PID 435) exited with exit code 1 logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:50.120 UTC [432] LOG: aborting startup due to startup process failure logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:50.120 UTC [432] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:50.170 UTC [432] LOG: database system is shut down logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.653 UTC [473] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.654 UTC [473] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.654 UTC [473] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.679 UTC [473] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.687 UTC [476] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.720 UTC [476] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.748 UTC [476] LOG: entering standby mode logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:26:59.777 UTC [476] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.108 UTC [476] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.144 UTC [476] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.145 UTC [473] LOG: startup process (PID 476) exited with exit code 1 logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.145 UTC [473] LOG: aborting startup due to startup process failure logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.145 UTC [473] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.194 UTC [473] LOG: database system is shut down logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.655 UTC [515] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.656 UTC [515] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.656 UTC [515] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.682 UTC [515] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.690 UTC [518] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.721 UTC [518] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.750 UTC [518] LOG: entering standby mode logger.go:42: 19:27:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.780 UTC [518] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:41 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 5m54s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 5m54s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 6m33s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 3m46s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 4m3s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 2m36s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 2m20s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 2m21s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 5m36s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 5m36s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 5m36s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 4m35s logger.go:42: 19:27:42 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 8m9s logger.go:42: 19:27:43 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:27:45 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.117 UTC [447] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:00.164 UTC [447] LOG: database system is shut down logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.629 UTC [496] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.630 UTC [496] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.630 UTC [496] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.662 UTC [496] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.669 UTC [507] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.699 UTC [507] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.728 UTC [507] LOG: entering standby mode logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:09.755 UTC [507] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:10.078 UTC [507] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:10.113 UTC [507] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:10.114 UTC [496] LOG: startup process (PID 507) exited with exit code 1 logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:10.114 UTC [496] LOG: aborting startup due to startup process failure logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:10.114 UTC [496] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:10.163 UTC [496] LOG: database system is shut down logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:29.617 UTC [561] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:29.618 UTC [561] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:29.618 UTC [561] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:29.644 UTC [561] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:29.653 UTC [573] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:29.682 UTC [573] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:29.708 UTC [573] LOG: entering standby mode logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:29.738 UTC [573] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:30.060 UTC [573] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:30.095 UTC [573] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:30.097 UTC [561] LOG: startup process (PID 573) exited with exit code 1 logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:30.097 UTC [561] LOG: aborting startup due to startup process failure logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:30.097 UTC [561] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:30.145 UTC [561] LOG: database system is shut down logger.go:42: 19:27:47 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:10.150 UTC [515] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:10.199 UTC [515] LOG: database system is shut down logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:19.644 UTC [565] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:19.645 UTC [565] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:19.645 UTC [565] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:19.671 UTC [565] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:19.680 UTC [568] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:19.711 UTC [568] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:19.738 UTC [568] LOG: entering standby mode logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:19.767 UTC [568] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:20.094 UTC [568] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:20.129 UTC [568] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:20.131 UTC [565] LOG: startup process (PID 568) exited with exit code 1 logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:20.131 UTC [565] LOG: aborting startup due to startup process failure logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:20.131 UTC [565] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:20.182 UTC [565] LOG: database system is shut down logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:39.649 UTC [631] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:39.650 UTC [631] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:39.650 UTC [631] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:39.675 UTC [631] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:39.683 UTC [634] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:39.712 UTC [634] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:39.741 UTC [634] LOG: entering standby mode logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:39.769 UTC [634] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:40.095 UTC [634] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:40.129 UTC [634] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:40.131 UTC [631] LOG: startup process (PID 634) exited with exit code 1 logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:40.131 UTC [631] LOG: aborting startup due to startup process failure logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:40.131 UTC [631] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:40.179 UTC [631] LOG: database system is shut down logger.go:42: 19:28:20 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 6m33s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 6m33s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 7m12s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 4m25s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 4m42s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 3m15s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 2m59s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 3m logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 6m15s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 6m15s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 6m15s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 5m14s logger.go:42: 19:28:21 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 8m48s logger.go:42: 19:28:22 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:28:24 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:30.097 UTC [561] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:30.145 UTC [561] LOG: database system is shut down logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.626 UTC [636] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.627 UTC [636] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.627 UTC [636] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.655 UTC [636] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.664 UTC [648] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.692 UTC [648] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.718 UTC [648] LOG: entering standby mode logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.749 UTC [648] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.071 UTC [648] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.105 UTC [648] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.106 UTC [636] LOG: startup process (PID 648) exited with exit code 1 logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.106 UTC [636] LOG: aborting startup due to startup process failure logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.106 UTC [636] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.155 UTC [636] LOG: database system is shut down logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.621 UTC [703] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.621 UTC [703] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.622 UTC [703] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.648 UTC [703] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.655 UTC [715] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.686 UTC [715] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.713 UTC [715] LOG: entering standby mode logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.743 UTC [715] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.066 UTC [715] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.102 UTC [715] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.103 UTC [703] LOG: startup process (PID 715) exited with exit code 1 logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.103 UTC [703] LOG: aborting startup due to startup process failure logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.103 UTC [703] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.150 UTC [703] LOG: database system is shut down logger.go:42: 19:28:27 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:40.131 UTC [631] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:40.179 UTC [631] LOG: database system is shut down logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.671 UTC [680] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.672 UTC [680] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.672 UTC [680] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.698 UTC [680] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.706 UTC [683] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.737 UTC [683] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.765 UTC [683] LOG: entering standby mode logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:49.796 UTC [683] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.130 UTC [683] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.165 UTC [683] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.167 UTC [680] LOG: startup process (PID 683) exited with exit code 1 logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.167 UTC [680] LOG: aborting startup due to startup process failure logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.167 UTC [680] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:27:50.216 UTC [680] LOG: database system is shut down logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.628 UTC [746] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.629 UTC [746] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.629 UTC [746] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.655 UTC [746] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.663 UTC [749] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.693 UTC [749] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.723 UTC [749] LOG: entering standby mode logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:09.753 UTC [749] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.077 UTC [749] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.113 UTC [749] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.114 UTC [746] LOG: startup process (PID 749) exited with exit code 1 logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.114 UTC [746] LOG: aborting startup due to startup process failure logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.114 UTC [746] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:10.165 UTC [746] LOG: database system is shut down logger.go:42: 19:28:59 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 7m13s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 7m13s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 7m52s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 5m5s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 5m22s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 3m55s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 3m39s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 3m40s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 6m55s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 6m55s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 6m55s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 5m54s logger.go:42: 19:29:00 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 9m28s logger.go:42: 19:29:02 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:29:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:29:04 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:30.093 UTC [787] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:30.140 UTC [787] LOG: database system is shut down logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.606 UTC [829] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.606 UTC [829] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.606 UTC [829] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.631 UTC [829] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.639 UTC [832] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.670 UTC [832] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.698 UTC [832] LOG: entering standby mode logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.724 UTC [832] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.051 UTC [832] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.084 UTC [832] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.086 UTC [829] LOG: startup process (PID 832) exited with exit code 1 logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.086 UTC [829] LOG: aborting startup due to startup process failure logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.086 UTC [829] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.134 UTC [829] LOG: database system is shut down logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.597 UTC [895] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.598 UTC [895] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.598 UTC [895] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.623 UTC [895] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.631 UTC [898] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.660 UTC [898] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.689 UTC [898] LOG: entering standby mode logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.715 UTC [898] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.040 UTC [898] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.074 UTC [898] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.076 UTC [895] LOG: startup process (PID 898) exited with exit code 1 logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.076 UTC [895] LOG: aborting startup due to startup process failure logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.076 UTC [895] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.123 UTC [895] LOG: database system is shut down logger.go:42: 19:29:06 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:30.119 UTC [820] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:30.170 UTC [820] LOG: database system is shut down logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.626 UTC [862] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.627 UTC [862] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.627 UTC [862] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.653 UTC [862] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.662 UTC [865] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.698 UTC [865] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.725 UTC [865] LOG: entering standby mode logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:39.755 UTC [865] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.080 UTC [865] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.115 UTC [865] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.116 UTC [862] LOG: startup process (PID 865) exited with exit code 1 logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.116 UTC [862] LOG: aborting startup due to startup process failure logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.116 UTC [862] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:40.167 UTC [862] LOG: database system is shut down logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.614 UTC [929] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.615 UTC [929] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.615 UTC [929] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.641 UTC [929] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.649 UTC [932] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.680 UTC [932] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.710 UTC [932] LOG: entering standby mode logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:28:59.742 UTC [932] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.066 UTC [932] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.101 UTC [932] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.102 UTC [929] LOG: startup process (PID 932) exited with exit code 1 logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.102 UTC [929] LOG: aborting startup due to startup process failure logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.102 UTC [929] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.154 UTC [929] LOG: database system is shut down logger.go:42: 19:29:38 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 7m52s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 7m52s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 8m31s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 5m44s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 6m1s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 4m34s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 4m18s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 4m19s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 7m34s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 7m34s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 7m34s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 6m33s logger.go:42: 19:29:39 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 10m logger.go:42: 19:29:41 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:29:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:29:43 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.076 UTC [895] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.123 UTC [895] LOG: database system is shut down logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.617 UTC [945] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.617 UTC [945] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.617 UTC [945] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.643 UTC [945] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.650 UTC [948] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.680 UTC [948] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.707 UTC [948] LOG: entering standby mode logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.734 UTC [948] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.058 UTC [948] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.092 UTC [948] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.094 UTC [945] LOG: startup process (PID 948) exited with exit code 1 logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.094 UTC [945] LOG: aborting startup due to startup process failure logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.094 UTC [945] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.142 UTC [945] LOG: database system is shut down logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.629 UTC [1011] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.630 UTC [1011] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.630 UTC [1011] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.656 UTC [1011] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.666 UTC [1014] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.693 UTC [1014] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.720 UTC [1014] LOG: entering standby mode logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.751 UTC [1014] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.074 UTC [1014] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.110 UTC [1014] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.111 UTC [1011] LOG: startup process (PID 1014) exited with exit code 1 logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.111 UTC [1011] LOG: aborting startup due to startup process failure logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.111 UTC [1011] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.159 UTC [1011] LOG: database system is shut down logger.go:42: 19:29:45 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.102 UTC [929] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:00.154 UTC [929] LOG: database system is shut down logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.616 UTC [979] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.616 UTC [979] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.616 UTC [979] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.642 UTC [979] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.649 UTC [982] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.681 UTC [982] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.711 UTC [982] LOG: entering standby mode logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:09.743 UTC [982] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.070 UTC [982] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.105 UTC [982] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.106 UTC [979] LOG: startup process (PID 982) exited with exit code 1 logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.106 UTC [979] LOG: aborting startup due to startup process failure logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.106 UTC [979] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:10.156 UTC [979] LOG: database system is shut down logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.618 UTC [1046] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.618 UTC [1046] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.618 UTC [1046] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.644 UTC [1046] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.651 UTC [1049] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.681 UTC [1049] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.710 UTC [1049] LOG: entering standby mode logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:29.740 UTC [1049] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.064 UTC [1049] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.099 UTC [1049] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.101 UTC [1046] LOG: startup process (PID 1049) exited with exit code 1 logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.101 UTC [1046] LOG: aborting startup due to startup process failure logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.101 UTC [1046] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:30.152 UTC [1046] LOG: database system is shut down logger.go:42: 19:30:17 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 8m31s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 8m31s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 9m10s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 6m23s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 6m40s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 5m13s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 4m57s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 4m58s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 8m13s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 8m13s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 8m13s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 7m12s logger.go:42: 19:30:18 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 10m logger.go:42: 19:30:20 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:30:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:30:22 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:50.095 UTC [1095] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:50.142 UTC [1095] LOG: database system is shut down logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.608 UTC [1162] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.608 UTC [1162] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.608 UTC [1162] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.644 UTC [1162] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.652 UTC [1165] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.683 UTC [1165] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.710 UTC [1165] LOG: entering standby mode logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.738 UTC [1165] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.063 UTC [1165] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.099 UTC [1165] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.100 UTC [1162] LOG: startup process (PID 1165) exited with exit code 1 logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.100 UTC [1162] LOG: aborting startup due to startup process failure logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.100 UTC [1162] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.147 UTC [1162] LOG: database system is shut down logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.604 UTC [1204] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.605 UTC [1204] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.605 UTC [1204] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.630 UTC [1204] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.638 UTC [1207] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.668 UTC [1207] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.696 UTC [1207] LOG: entering standby mode logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.723 UTC [1207] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.045 UTC [1207] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.080 UTC [1207] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.082 UTC [1204] LOG: startup process (PID 1207) exited with exit code 1 logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.082 UTC [1204] LOG: aborting startup due to startup process failure logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.082 UTC [1204] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.130 UTC [1204] LOG: database system is shut down logger.go:42: 19:30:24 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:50.102 UTC [1121] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:29:50.152 UTC [1121] LOG: database system is shut down logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.615 UTC [1189] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.615 UTC [1189] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.616 UTC [1189] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.641 UTC [1189] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.649 UTC [1192] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.682 UTC [1192] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.711 UTC [1192] LOG: entering standby mode logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:09.741 UTC [1192] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.065 UTC [1192] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.101 UTC [1192] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.103 UTC [1189] LOG: startup process (PID 1192) exited with exit code 1 logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.103 UTC [1189] LOG: aborting startup due to startup process failure logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.103 UTC [1189] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:10.152 UTC [1189] LOG: database system is shut down logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.611 UTC [1232] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.611 UTC [1232] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.611 UTC [1232] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.637 UTC [1232] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.645 UTC [1235] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.677 UTC [1235] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.706 UTC [1235] LOG: entering standby mode logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:19.735 UTC [1235] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.058 UTC [1235] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.093 UTC [1235] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.095 UTC [1232] LOG: startup process (PID 1235) exited with exit code 1 logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.095 UTC [1232] LOG: aborting startup due to startup process failure logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.095 UTC [1232] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.145 UTC [1232] LOG: database system is shut down logger.go:42: 19:30:56 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 9m10s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 9m10s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 9m49s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 7m2s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 7m19s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 5m52s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 5m36s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 5m37s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 8m52s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 8m52s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 8m52s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 7m51s logger.go:42: 19:30:57 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 11m logger.go:42: 19:30:59 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:31:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:31:01 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.082 UTC [1204] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.130 UTC [1204] LOG: database system is shut down logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.612 UTC [1279] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.612 UTC [1279] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.612 UTC [1279] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.638 UTC [1279] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.645 UTC [1282] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.677 UTC [1282] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.705 UTC [1282] LOG: entering standby mode logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.733 UTC [1282] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.056 UTC [1282] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.090 UTC [1282] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.091 UTC [1279] LOG: startup process (PID 1282) exited with exit code 1 logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.091 UTC [1279] LOG: aborting startup due to startup process failure logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.091 UTC [1279] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.139 UTC [1279] LOG: database system is shut down logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.606 UTC [1321] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.607 UTC [1321] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.607 UTC [1321] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.631 UTC [1321] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.639 UTC [1324] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.669 UTC [1324] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.696 UTC [1324] LOG: entering standby mode logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.721 UTC [1324] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.042 UTC [1324] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.077 UTC [1324] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.079 UTC [1321] LOG: startup process (PID 1324) exited with exit code 1 logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.079 UTC [1321] LOG: aborting startup due to startup process failure logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.079 UTC [1321] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.126 UTC [1321] LOG: database system is shut down logger.go:42: 19:31:03 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.095 UTC [1232] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:20.145 UTC [1232] LOG: database system is shut down logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.613 UTC [1307] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.614 UTC [1307] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.614 UTC [1307] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.640 UTC [1307] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.649 UTC [1310] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.679 UTC [1310] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.708 UTC [1310] LOG: entering standby mode logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:39.737 UTC [1310] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.063 UTC [1310] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.098 UTC [1310] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.100 UTC [1307] LOG: startup process (PID 1310) exited with exit code 1 logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.100 UTC [1307] LOG: aborting startup due to startup process failure logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.100 UTC [1307] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:40.150 UTC [1307] LOG: database system is shut down logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.615 UTC [1349] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.616 UTC [1349] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.616 UTC [1349] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.641 UTC [1349] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.649 UTC [1352] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.679 UTC [1352] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.706 UTC [1352] LOG: entering standby mode logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:49.735 UTC [1352] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.057 UTC [1352] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.092 UTC [1352] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.094 UTC [1349] LOG: startup process (PID 1352) exited with exit code 1 logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.094 UTC [1349] LOG: aborting startup due to startup process failure logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.094 UTC [1349] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.144 UTC [1349] LOG: database system is shut down logger.go:42: 19:31:35 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 9m49s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 9m49s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 10m logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 7m41s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 7m58s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 6m31s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 6m15s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 6m16s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 9m31s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 9m31s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 9m31s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 8m30s logger.go:42: 19:31:36 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 12m logger.go:42: 19:31:38 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:31:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:31:40 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:10.087 UTC [1396] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:10.134 UTC [1396] LOG: database system is shut down logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.588 UTC [1438] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.589 UTC [1438] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.589 UTC [1438] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.614 UTC [1438] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.622 UTC [1441] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.656 UTC [1441] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.683 UTC [1441] LOG: entering standby mode logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.709 UTC [1441] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.831 UTC [1441] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.867 UTC [1441] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.869 UTC [1438] LOG: startup process (PID 1441) exited with exit code 1 logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.869 UTC [1438] LOG: aborting startup due to startup process failure logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.869 UTC [1438] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:19.921 UTC [1438] LOG: database system is shut down logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.602 UTC [1480] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.603 UTC [1480] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.603 UTC [1480] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.628 UTC [1480] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.636 UTC [1483] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.668 UTC [1483] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.694 UTC [1483] LOG: entering standby mode logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.720 UTC [1483] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.042 UTC [1483] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.077 UTC [1483] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.078 UTC [1480] LOG: startup process (PID 1483) exited with exit code 1 logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.078 UTC [1480] LOG: aborting startup due to startup process failure logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.078 UTC [1480] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.128 UTC [1480] LOG: database system is shut down logger.go:42: 19:31:42 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.094 UTC [1349] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:30:50.144 UTC [1349] LOG: database system is shut down logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:09.620 UTC [1423] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:09.620 UTC [1423] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:09.620 UTC [1423] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:09.647 UTC [1423] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:09.654 UTC [1426] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:09.685 UTC [1426] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:09.714 UTC [1426] LOG: entering standby mode logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:09.742 UTC [1426] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:10.067 UTC [1426] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:10.101 UTC [1426] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:10.103 UTC [1423] LOG: startup process (PID 1426) exited with exit code 1 logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:10.103 UTC [1423] LOG: aborting startup due to startup process failure logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:10.103 UTC [1423] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:10.153 UTC [1423] LOG: database system is shut down logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.621 UTC [1499] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.622 UTC [1499] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.622 UTC [1499] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.648 UTC [1499] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.655 UTC [1502] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.686 UTC [1502] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.714 UTC [1502] LOG: entering standby mode logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:29.742 UTC [1502] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.064 UTC [1502] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.100 UTC [1502] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.102 UTC [1499] LOG: startup process (PID 1502) exited with exit code 1 logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.102 UTC [1499] LOG: aborting startup due to startup process failure logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.102 UTC [1499] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.152 UTC [1499] LOG: database system is shut down logger.go:42: 19:32:14 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 10m logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 10m logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 11m logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 8m20s logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 8m37s logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 7m10s logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 6m54s logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 6m55s logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 10m logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 10m logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 10m logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 9m9s logger.go:42: 19:32:15 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 12m logger.go:42: 19:32:17 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:32:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:32:19 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.078 UTC [1480] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.128 UTC [1480] LOG: database system is shut down logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.631 UTC [1555] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.632 UTC [1555] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.632 UTC [1555] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.657 UTC [1555] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.668 UTC [1558] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.696 UTC [1558] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.726 UTC [1558] LOG: entering standby mode logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.755 UTC [1558] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.081 UTC [1558] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.115 UTC [1558] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.117 UTC [1555] LOG: startup process (PID 1558) exited with exit code 1 logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.117 UTC [1555] LOG: aborting startup due to startup process failure logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.117 UTC [1555] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.165 UTC [1555] LOG: database system is shut down logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.600 UTC [1622] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.601 UTC [1622] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.601 UTC [1622] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.627 UTC [1622] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.634 UTC [1625] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.664 UTC [1625] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.691 UTC [1625] LOG: entering standby mode logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.718 UTC [1625] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.040 UTC [1625] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.075 UTC [1625] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.076 UTC [1622] LOG: startup process (PID 1625) exited with exit code 1 logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.076 UTC [1622] LOG: aborting startup due to startup process failure logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.076 UTC [1622] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.124 UTC [1622] LOG: database system is shut down logger.go:42: 19:32:21 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.102 UTC [1499] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:30.152 UTC [1499] LOG: database system is shut down logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.637 UTC [1572] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.638 UTC [1572] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.638 UTC [1572] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.664 UTC [1572] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.672 UTC [1575] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.703 UTC [1575] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.730 UTC [1575] LOG: entering standby mode logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:49.759 UTC [1575] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.083 UTC [1575] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.118 UTC [1575] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.119 UTC [1572] LOG: startup process (PID 1575) exited with exit code 1 logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.119 UTC [1572] LOG: aborting startup due to startup process failure logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.119 UTC [1572] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:31:50.169 UTC [1572] LOG: database system is shut down logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.614 UTC [1638] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.615 UTC [1638] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.615 UTC [1638] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.641 UTC [1638] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.648 UTC [1641] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.680 UTC [1641] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.709 UTC [1641] LOG: entering standby mode logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:09.737 UTC [1641] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.063 UTC [1641] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.098 UTC [1641] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.100 UTC [1638] LOG: startup process (PID 1641) exited with exit code 1 logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.100 UTC [1638] LOG: aborting startup due to startup process failure logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.100 UTC [1638] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.151 UTC [1638] LOG: database system is shut down logger.go:42: 19:32:53 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 11m logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 11m logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 11m logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 8m59s logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 9m16s logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 7m49s logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 7m33s logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 7m34s logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 10m logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 10m logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 10m logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 9m48s logger.go:42: 19:32:54 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 13m logger.go:42: 19:32:56 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:32:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:32:58 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.076 UTC [1622] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.124 UTC [1622] LOG: database system is shut down logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.606 UTC [1706] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.607 UTC [1706] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.607 UTC [1706] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.634 UTC [1706] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.642 UTC [1709] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.672 UTC [1709] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.699 UTC [1709] LOG: entering standby mode logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.726 UTC [1709] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.048 UTC [1709] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.082 UTC [1709] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.084 UTC [1706] LOG: startup process (PID 1709) exited with exit code 1 logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.084 UTC [1706] LOG: aborting startup due to startup process failure logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.084 UTC [1706] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.133 UTC [1706] LOG: database system is shut down logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:39.665 UTC [1748] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:39.666 UTC [1748] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:39.666 UTC [1748] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:39.691 UTC [1748] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:39.698 UTC [1751] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:39.724 UTC [1751] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:39.751 UTC [1751] LOG: entering standby mode logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:39.778 UTC [1751] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:40.101 UTC [1751] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:40.140 UTC [1751] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:40.142 UTC [1748] LOG: startup process (PID 1751) exited with exit code 1 logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:40.142 UTC [1748] LOG: aborting startup due to startup process failure logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:40.142 UTC [1748] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:40.205 UTC [1748] LOG: database system is shut down logger.go:42: 19:33:00 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.100 UTC [1638] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:10.151 UTC [1638] LOG: database system is shut down logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.625 UTC [1713] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.625 UTC [1713] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.625 UTC [1713] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.651 UTC [1713] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.659 UTC [1716] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.689 UTC [1716] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.718 UTC [1716] LOG: entering standby mode logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:29.747 UTC [1716] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.073 UTC [1716] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.109 UTC [1716] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.111 UTC [1713] LOG: startup process (PID 1716) exited with exit code 1 logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.111 UTC [1713] LOG: aborting startup due to startup process failure logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.111 UTC [1713] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:30.161 UTC [1713] LOG: database system is shut down logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:49.628 UTC [1781] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:49.629 UTC [1781] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:49.629 UTC [1781] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:49.654 UTC [1781] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:49.663 UTC [1784] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:49.694 UTC [1784] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:49.724 UTC [1784] LOG: entering standby mode logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:49.752 UTC [1784] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:50.076 UTC [1784] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:50.112 UTC [1784] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:50.113 UTC [1781] LOG: startup process (PID 1784) exited with exit code 1 logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:50.113 UTC [1781] LOG: aborting startup due to startup process failure logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:50.113 UTC [1781] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:32:50.164 UTC [1781] LOG: database system is shut down logger.go:42: 19:33:32 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 11m logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 11m logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 12m logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 9m38s logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 9m55s logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 8m28s logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 8m12s logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 8m13s logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 11m logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 11m logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 11m logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 10m logger.go:42: 19:33:33 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 14m logger.go:42: 19:33:35 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:33:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:33:37 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:00.106 UTC [1815] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:00.156 UTC [1815] LOG: database system is shut down logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:09.622 UTC [1865] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:09.623 UTC [1865] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:09.623 UTC [1865] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:09.667 UTC [1865] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:09.675 UTC [1868] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:09.704 UTC [1868] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:09.731 UTC [1868] LOG: entering standby mode logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:09.757 UTC [1868] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:10.079 UTC [1868] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:10.114 UTC [1868] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:10.115 UTC [1865] LOG: startup process (PID 1868) exited with exit code 1 logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:10.116 UTC [1865] LOG: aborting startup due to startup process failure logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:10.116 UTC [1865] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:10.165 UTC [1865] LOG: database system is shut down logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.595 UTC [1932] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.595 UTC [1932] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.595 UTC [1932] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.621 UTC [1932] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.629 UTC [1935] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.659 UTC [1935] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.688 UTC [1935] LOG: entering standby mode logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.715 UTC [1935] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.036 UTC [1935] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.071 UTC [1935] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.072 UTC [1932] LOG: startup process (PID 1935) exited with exit code 1 logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.072 UTC [1932] LOG: aborting startup due to startup process failure logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.073 UTC [1932] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.121 UTC [1932] LOG: database system is shut down logger.go:42: 19:33:39 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:10.092 UTC [1854] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:10.143 UTC [1854] LOG: database system is shut down logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.601 UTC [1922] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.602 UTC [1922] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.602 UTC [1922] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.627 UTC [1922] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.635 UTC [1925] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.667 UTC [1925] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.696 UTC [1925] LOG: entering standby mode logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:29.724 UTC [1925] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.047 UTC [1925] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.082 UTC [1925] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.084 UTC [1922] LOG: startup process (PID 1925) exited with exit code 1 logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.084 UTC [1922] LOG: aborting startup due to startup process failure logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.084 UTC [1922] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.134 UTC [1922] LOG: database system is shut down logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:39.615 UTC [1964] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:39.616 UTC [1964] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:39.616 UTC [1964] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:39.641 UTC [1964] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:39.649 UTC [1967] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:39.679 UTC [1967] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:39.707 UTC [1967] LOG: entering standby mode logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:39.735 UTC [1967] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:40.061 UTC [1967] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:40.096 UTC [1967] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:40.097 UTC [1964] LOG: startup process (PID 1967) exited with exit code 1 logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:40.097 UTC [1964] LOG: aborting startup due to startup process failure logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:40.097 UTC [1964] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:40.148 UTC [1964] LOG: database system is shut down logger.go:42: 19:34:11 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 12m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 12m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 13m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 10m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 10m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 9m7s logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 8m51s logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 8m52s logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 12m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 12m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 12m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 11m logger.go:42: 19:34:12 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 14m logger.go:42: 19:34:14 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:34:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:34:16 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.073 UTC [1932] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:30.121 UTC [1932] LOG: database system is shut down logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:49.620 UTC [2007] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:49.621 UTC [2007] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:49.621 UTC [2007] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:49.650 UTC [2007] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:49.658 UTC [2020] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:49.685 UTC [2020] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:49.713 UTC [2020] LOG: entering standby mode logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:49.739 UTC [2020] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:50.060 UTC [2020] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:50.094 UTC [2020] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:50.096 UTC [2007] LOG: startup process (PID 2020) exited with exit code 1 logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:50.096 UTC [2007] LOG: aborting startup due to startup process failure logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:50.096 UTC [2007] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:50.146 UTC [2007] LOG: database system is shut down logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.630 UTC [2075] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.631 UTC [2075] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.631 UTC [2075] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.656 UTC [2075] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.664 UTC [2087] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.692 UTC [2087] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.718 UTC [2087] LOG: entering standby mode logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.746 UTC [2087] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.068 UTC [2087] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.102 UTC [2087] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.104 UTC [2075] LOG: startup process (PID 2087) exited with exit code 1 logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.104 UTC [2075] LOG: aborting startup due to startup process failure logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.104 UTC [2075] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.155 UTC [2075] LOG: database system is shut down logger.go:42: 19:34:18 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:40.097 UTC [1964] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:40.148 UTC [1964] LOG: database system is shut down logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:59.633 UTC [2048] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:59.634 UTC [2048] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:59.634 UTC [2048] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:59.660 UTC [2048] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:59.669 UTC [2051] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:59.701 UTC [2051] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:59.729 UTC [2051] LOG: entering standby mode logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:33:59.759 UTC [2051] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:00.083 UTC [2051] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:00.119 UTC [2051] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:00.120 UTC [2048] LOG: startup process (PID 2051) exited with exit code 1 logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:00.120 UTC [2048] LOG: aborting startup due to startup process failure logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:00.120 UTC [2048] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:00.171 UTC [2048] LOG: database system is shut down logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.623 UTC [2091] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.623 UTC [2091] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.623 UTC [2091] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.649 UTC [2091] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.657 UTC [2094] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.689 UTC [2094] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.717 UTC [2094] LOG: entering standby mode logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:09.746 UTC [2094] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.069 UTC [2094] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.105 UTC [2094] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.106 UTC [2091] LOG: startup process (PID 2094) exited with exit code 1 logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.106 UTC [2091] LOG: aborting startup due to startup process failure logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.106 UTC [2091] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.157 UTC [2091] LOG: database system is shut down logger.go:42: 19:34:50 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 13m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 13m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 13m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 10m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 11m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 9m46s logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 9m30s logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 9m31s logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 12m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 12m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 12m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 11m logger.go:42: 19:34:51 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 15m logger.go:42: 19:34:52 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:34:54 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.104 UTC [2075] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.155 UTC [2075] LOG: database system is shut down logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.626 UTC [2150] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.627 UTC [2150] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.627 UTC [2150] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.652 UTC [2150] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.660 UTC [2162] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.688 UTC [2162] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.716 UTC [2162] LOG: entering standby mode logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.743 UTC [2162] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.065 UTC [2162] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.099 UTC [2162] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.101 UTC [2150] LOG: startup process (PID 2162) exited with exit code 1 logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.101 UTC [2150] LOG: aborting startup due to startup process failure logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.101 UTC [2150] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.150 UTC [2150] LOG: database system is shut down logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.633 UTC [2217] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.633 UTC [2217] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.634 UTC [2217] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.659 UTC [2217] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.668 UTC [2229] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.697 UTC [2229] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.724 UTC [2229] LOG: entering standby mode logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.750 UTC [2229] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.073 UTC [2229] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.107 UTC [2229] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.109 UTC [2217] LOG: startup process (PID 2229) exited with exit code 1 logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.109 UTC [2217] LOG: aborting startup due to startup process failure logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.109 UTC [2217] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.158 UTC [2217] LOG: database system is shut down logger.go:42: 19:34:57 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.106 UTC [2091] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:10.157 UTC [2091] LOG: database system is shut down logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.619 UTC [2166] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.619 UTC [2166] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.619 UTC [2166] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.645 UTC [2166] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.653 UTC [2169] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.685 UTC [2169] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.712 UTC [2169] LOG: entering standby mode logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:29.742 UTC [2169] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.067 UTC [2169] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.102 UTC [2169] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.104 UTC [2166] LOG: startup process (PID 2169) exited with exit code 1 logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.104 UTC [2166] LOG: aborting startup due to startup process failure logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.104 UTC [2166] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:30.154 UTC [2166] LOG: database system is shut down logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.609 UTC [2233] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.609 UTC [2233] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.609 UTC [2233] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.635 UTC [2233] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.643 UTC [2236] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.674 UTC [2236] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.703 UTC [2236] LOG: entering standby mode logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:49.732 UTC [2236] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.054 UTC [2236] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.089 UTC [2236] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.091 UTC [2233] LOG: startup process (PID 2236) exited with exit code 1 logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.091 UTC [2233] LOG: aborting startup due to startup process failure logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.091 UTC [2233] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.141 UTC [2233] LOG: database system is shut down logger.go:42: 19:35:29 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 13m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 13m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 14m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 11m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 11m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 10m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 10m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 10m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 13m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 13m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 13m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 12m logger.go:42: 19:35:30 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 15m logger.go:42: 19:35:31 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:35:33 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.109 UTC [2217] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.158 UTC [2217] LOG: database system is shut down logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.641 UTC [2293] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.641 UTC [2293] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.641 UTC [2293] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.667 UTC [2293] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.674 UTC [2314] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.704 UTC [2314] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.732 UTC [2314] LOG: entering standby mode logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.758 UTC [2314] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.081 UTC [2314] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.114 UTC [2314] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.116 UTC [2293] LOG: startup process (PID 2314) exited with exit code 1 logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.116 UTC [2293] LOG: aborting startup due to startup process failure logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.116 UTC [2293] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.164 UTC [2293] LOG: database system is shut down logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.609 UTC [2344] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.610 UTC [2344] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.610 UTC [2344] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.638 UTC [2344] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.647 UTC [2353] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.684 UTC [2353] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.711 UTC [2353] LOG: entering standby mode logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.740 UTC [2353] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.063 UTC [2353] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.098 UTC [2353] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.099 UTC [2344] LOG: startup process (PID 2353) exited with exit code 1 logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.099 UTC [2344] LOG: aborting startup due to startup process failure logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.099 UTC [2344] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.148 UTC [2344] LOG: database system is shut down logger.go:42: 19:35:36 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.091 UTC [2233] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:34:50.141 UTC [2233] LOG: database system is shut down logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.600 UTC [2308] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.600 UTC [2308] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.600 UTC [2308] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.625 UTC [2308] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.633 UTC [2311] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.668 UTC [2311] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.696 UTC [2311] LOG: entering standby mode logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:09.724 UTC [2311] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.049 UTC [2311] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.085 UTC [2311] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.086 UTC [2308] LOG: startup process (PID 2311) exited with exit code 1 logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.086 UTC [2308] LOG: aborting startup due to startup process failure logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.086 UTC [2308] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:10.136 UTC [2308] LOG: database system is shut down logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.624 UTC [2350] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.624 UTC [2350] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.624 UTC [2350] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.650 UTC [2350] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.658 UTC [2353] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.691 UTC [2353] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.721 UTC [2353] LOG: entering standby mode logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:19.750 UTC [2353] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.074 UTC [2353] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.109 UTC [2353] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.110 UTC [2350] LOG: startup process (PID 2353) exited with exit code 1 logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.110 UTC [2350] LOG: aborting startup due to startup process failure logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.110 UTC [2350] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:20.160 UTC [2350] LOG: database system is shut down logger.go:42: 19:36:08 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 14m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 14m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 15m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 12m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 12m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 11m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 10m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 10m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 14m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 14m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 14m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 13m logger.go:42: 19:36:09 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 16m logger.go:42: 19:36:10 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:36:12 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:40.103 UTC [2418] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:40.153 UTC [2418] LOG: database system is shut down logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.608 UTC [2485] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.609 UTC [2485] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.609 UTC [2485] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.636 UTC [2485] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.644 UTC [2497] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.678 UTC [2497] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.706 UTC [2497] LOG: entering standby mode logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.734 UTC [2497] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.056 UTC [2497] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.089 UTC [2497] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.091 UTC [2485] LOG: startup process (PID 2497) exited with exit code 1 logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.091 UTC [2485] LOG: aborting startup due to startup process failure logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.091 UTC [2485] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.140 UTC [2485] LOG: database system is shut down logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.633 UTC [2526] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.633 UTC [2526] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.633 UTC [2526] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.665 UTC [2526] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.675 UTC [2538] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.708 UTC [2538] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.734 UTC [2538] LOG: entering standby mode logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.761 UTC [2538] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.083 UTC [2538] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.118 UTC [2538] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.119 UTC [2526] LOG: startup process (PID 2538) exited with exit code 1 logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.119 UTC [2526] LOG: aborting startup due to startup process failure logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.119 UTC [2526] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.169 UTC [2526] LOG: database system is shut down logger.go:42: 19:36:15 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:40.107 UTC [2425] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:40.159 UTC [2425] LOG: database system is shut down logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.603 UTC [2491] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.603 UTC [2491] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.603 UTC [2491] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.629 UTC [2491] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.637 UTC [2494] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.668 UTC [2494] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.696 UTC [2494] LOG: entering standby mode logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:35:59.725 UTC [2494] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.050 UTC [2494] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.085 UTC [2494] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.086 UTC [2491] LOG: startup process (PID 2494) exited with exit code 1 logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.086 UTC [2491] LOG: aborting startup due to startup process failure logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.086 UTC [2491] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:00.136 UTC [2491] LOG: database system is shut down logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.629 UTC [2533] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.629 UTC [2533] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.629 UTC [2533] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.656 UTC [2533] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.664 UTC [2536] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.696 UTC [2536] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.723 UTC [2536] LOG: entering standby mode logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:09.753 UTC [2536] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.076 UTC [2536] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.111 UTC [2536] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.112 UTC [2533] LOG: startup process (PID 2536) exited with exit code 1 logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.112 UTC [2533] LOG: aborting startup due to startup process failure logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.112 UTC [2533] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.162 UTC [2533] LOG: database system is shut down logger.go:42: 19:36:47 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 15m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 15m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 15m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 12m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 13m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 11m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 11m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 11m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 14m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 14m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 14m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 13m logger.go:42: 19:36:48 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 17m logger.go:42: 19:36:50 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:36:51 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:36:52 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.119 UTC [2526] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.169 UTC [2526] LOG: database system is shut down logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.622 UTC [2599] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.622 UTC [2599] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.622 UTC [2599] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.653 UTC [2599] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.665 UTC [2611] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.695 UTC [2611] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.724 UTC [2611] LOG: entering standby mode logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.751 UTC [2611] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.073 UTC [2611] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.108 UTC [2611] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.109 UTC [2599] LOG: startup process (PID 2611) exited with exit code 1 logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.109 UTC [2599] LOG: aborting startup due to startup process failure logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.109 UTC [2599] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.158 UTC [2599] LOG: database system is shut down logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.622 UTC [2650] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.623 UTC [2650] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.623 UTC [2650] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.648 UTC [2650] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.657 UTC [2653] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.687 UTC [2653] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.713 UTC [2653] LOG: entering standby mode logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.740 UTC [2653] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.062 UTC [2653] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.098 UTC [2653] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.099 UTC [2650] LOG: startup process (PID 2653) exited with exit code 1 logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.099 UTC [2650] LOG: aborting startup due to startup process failure logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.099 UTC [2650] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.149 UTC [2650] LOG: database system is shut down logger.go:42: 19:36:53 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.112 UTC [2533] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:10.162 UTC [2533] LOG: database system is shut down logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.605 UTC [2608] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.606 UTC [2608] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.606 UTC [2608] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.632 UTC [2608] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.639 UTC [2611] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.673 UTC [2611] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.700 UTC [2611] LOG: entering standby mode logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:29.728 UTC [2611] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.054 UTC [2611] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.089 UTC [2611] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.090 UTC [2608] LOG: startup process (PID 2611) exited with exit code 1 logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.090 UTC [2608] LOG: aborting startup due to startup process failure logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.090 UTC [2608] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:30.141 UTC [2608] LOG: database system is shut down logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.616 UTC [2650] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.617 UTC [2650] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.617 UTC [2650] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.643 UTC [2650] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.651 UTC [2653] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.685 UTC [2653] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.713 UTC [2653] LOG: entering standby mode logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:39.742 UTC [2653] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.071 UTC [2653] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.105 UTC [2653] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.107 UTC [2650] LOG: startup process (PID 2653) exited with exit code 1 logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.107 UTC [2650] LOG: aborting startup due to startup process failure logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.107 UTC [2650] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:55 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:36:40.157 UTC [2650] LOG: database system is shut down logger.go:42: 19:37:26 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 15m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 15m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 16m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 13m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 13m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 12m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 12m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 12m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 15m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 15m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 15m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 14m logger.go:42: 19:37:27 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 17m logger.go:42: 19:37:28 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:37:30 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:00.097 UTC [2725] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:00.147 UTC [2725] LOG: database system is shut down logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.598 UTC [2791] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.599 UTC [2791] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.599 UTC [2791] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.624 UTC [2791] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.636 UTC [2794] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.665 UTC [2794] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.694 UTC [2794] LOG: entering standby mode logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.721 UTC [2794] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.045 UTC [2794] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.080 UTC [2794] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.082 UTC [2791] LOG: startup process (PID 2794) exited with exit code 1 logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.082 UTC [2791] LOG: aborting startup due to startup process failure logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.082 UTC [2791] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.130 UTC [2791] LOG: database system is shut down logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.608 UTC [2833] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.609 UTC [2833] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.609 UTC [2833] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.635 UTC [2833] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.642 UTC [2836] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.672 UTC [2836] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.700 UTC [2836] LOG: entering standby mode logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.728 UTC [2836] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.050 UTC [2836] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.084 UTC [2836] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.086 UTC [2833] LOG: startup process (PID 2836) exited with exit code 1 logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.086 UTC [2833] LOG: aborting startup due to startup process failure logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.086 UTC [2833] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.135 UTC [2833] LOG: database system is shut down logger.go:42: 19:37:32 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:00.103 UTC [2733] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:00.153 UTC [2733] LOG: database system is shut down logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.614 UTC [2800] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.615 UTC [2800] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.615 UTC [2800] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.641 UTC [2800] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.648 UTC [2803] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.681 UTC [2803] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.710 UTC [2803] LOG: entering standby mode logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:19.737 UTC [2803] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.060 UTC [2803] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.096 UTC [2803] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.097 UTC [2800] LOG: startup process (PID 2803) exited with exit code 1 logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.098 UTC [2800] LOG: aborting startup due to startup process failure logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.098 UTC [2800] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:20.147 UTC [2800] LOG: database system is shut down logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.617 UTC [2842] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.618 UTC [2842] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.618 UTC [2842] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.643 UTC [2842] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.650 UTC [2845] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.683 UTC [2845] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.712 UTC [2845] LOG: entering standby mode logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:29.741 UTC [2845] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.064 UTC [2845] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.099 UTC [2845] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.101 UTC [2842] LOG: startup process (PID 2845) exited with exit code 1 logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.101 UTC [2842] LOG: aborting startup due to startup process failure logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.101 UTC [2842] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:34 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.151 UTC [2842] LOG: database system is shut down logger.go:42: 19:38:05 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 16m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 16m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 16m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 14m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 14m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 13m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 12m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 12m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 16m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 16m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 16m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 14m logger.go:42: 19:38:06 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 18m logger.go:42: 19:38:07 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:38:09 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.086 UTC [2833] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.135 UTC [2833] LOG: database system is shut down logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.634 UTC [2917] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.635 UTC [2917] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.635 UTC [2917] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.665 UTC [2917] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.673 UTC [2920] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.702 UTC [2920] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.729 UTC [2920] LOG: entering standby mode logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.756 UTC [2920] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.080 UTC [2920] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.115 UTC [2920] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.117 UTC [2917] LOG: startup process (PID 2920) exited with exit code 1 logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.117 UTC [2917] LOG: aborting startup due to startup process failure logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.117 UTC [2917] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.167 UTC [2917] LOG: database system is shut down logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.609 UTC [2959] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.610 UTC [2959] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.610 UTC [2959] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.635 UTC [2959] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.642 UTC [2962] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.672 UTC [2962] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.699 UTC [2962] LOG: entering standby mode logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.727 UTC [2962] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.049 UTC [2962] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.083 UTC [2962] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.085 UTC [2959] LOG: startup process (PID 2962) exited with exit code 1 logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.085 UTC [2959] LOG: aborting startup due to startup process failure logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.085 UTC [2959] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:11 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.133 UTC [2959] LOG: database system is shut down logger.go:42: 19:38:12 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.101 UTC [2842] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:30.151 UTC [2842] LOG: database system is shut down logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.617 UTC [2917] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.618 UTC [2917] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.618 UTC [2917] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.644 UTC [2917] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.652 UTC [2920] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.685 UTC [2920] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.713 UTC [2920] LOG: entering standby mode logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:49.742 UTC [2920] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.065 UTC [2920] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.101 UTC [2920] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.103 UTC [2917] LOG: startup process (PID 2920) exited with exit code 1 logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.103 UTC [2917] LOG: aborting startup due to startup process failure logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.103 UTC [2917] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:50.153 UTC [2917] LOG: database system is shut down logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.630 UTC [2958] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.630 UTC [2958] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.630 UTC [2958] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.656 UTC [2958] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.664 UTC [2961] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.695 UTC [2961] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.723 UTC [2961] LOG: entering standby mode logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:37:59.753 UTC [2961] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.076 UTC [2961] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.111 UTC [2961] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.113 UTC [2958] LOG: startup process (PID 2961) exited with exit code 1 logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.113 UTC [2958] LOG: aborting startup due to startup process failure logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.113 UTC [2958] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:13 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.163 UTC [2958] LOG: database system is shut down logger.go:42: 19:38:44 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 16m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 16m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 17m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 14m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 15m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 13m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 13m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 13m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 16m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 16m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 16m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 15m logger.go:42: 19:38:45 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 19m logger.go:42: 19:38:47 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:38:48 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:38:49 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.085 UTC [2959] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.133 UTC [2959] LOG: database system is shut down logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.605 UTC [3034] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.606 UTC [3034] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.606 UTC [3034] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.631 UTC [3034] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.638 UTC [3037] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.671 UTC [3037] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.699 UTC [3037] LOG: entering standby mode logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.726 UTC [3037] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.049 UTC [3037] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.083 UTC [3037] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.085 UTC [3034] LOG: startup process (PID 3037) exited with exit code 1 logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.085 UTC [3034] LOG: aborting startup due to startup process failure logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.085 UTC [3034] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.134 UTC [3034] LOG: database system is shut down logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.607 UTC [3101] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.608 UTC [3101] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.608 UTC [3101] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.633 UTC [3101] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.640 UTC [3104] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.671 UTC [3104] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.698 UTC [3104] LOG: entering standby mode logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.725 UTC [3104] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.047 UTC [3104] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.082 UTC [3104] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.083 UTC [3101] LOG: startup process (PID 3104) exited with exit code 1 logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.083 UTC [3101] LOG: aborting startup due to startup process failure logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.083 UTC [3101] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:50 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.132 UTC [3101] LOG: database system is shut down logger.go:42: 19:38:51 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.113 UTC [2958] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:00.163 UTC [2958] LOG: database system is shut down logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.609 UTC [3042] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.610 UTC [3042] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.610 UTC [3042] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.636 UTC [3042] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.644 UTC [3045] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.677 UTC [3045] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.706 UTC [3045] LOG: entering standby mode logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:19.735 UTC [3045] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.059 UTC [3045] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.094 UTC [3045] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.095 UTC [3042] LOG: startup process (PID 3045) exited with exit code 1 logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.095 UTC [3042] LOG: aborting startup due to startup process failure logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.095 UTC [3042] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:20.147 UTC [3042] LOG: database system is shut down logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.615 UTC [3109] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.615 UTC [3109] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.615 UTC [3109] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.641 UTC [3109] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.648 UTC [3112] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.680 UTC [3112] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.707 UTC [3112] LOG: entering standby mode logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:39.736 UTC [3112] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.063 UTC [3112] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.099 UTC [3112] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.100 UTC [3109] LOG: startup process (PID 3112) exited with exit code 1 logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.100 UTC [3109] LOG: aborting startup due to startup process failure logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.100 UTC [3109] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:52 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.151 UTC [3109] LOG: database system is shut down logger.go:42: 19:39:23 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 17m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 17m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 18m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 15m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 15m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 14m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 14m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 14m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 17m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 17m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 17m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 16m logger.go:42: 19:39:24 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 19m logger.go:42: 19:39:26 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:39:27 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:39:28 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.083 UTC [3101] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:40.132 UTC [3101] LOG: database system is shut down logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:59.609 UTC [3174] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:59.610 UTC [3174] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:59.610 UTC [3174] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:59.635 UTC [3174] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:59.643 UTC [3177] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:59.672 UTC [3177] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:59.700 UTC [3177] LOG: entering standby mode logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:38:59.727 UTC [3177] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:00.050 UTC [3177] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:00.083 UTC [3177] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:00.085 UTC [3174] LOG: startup process (PID 3177) exited with exit code 1 logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:00.085 UTC [3174] LOG: aborting startup due to startup process failure logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:00.085 UTC [3174] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:00.134 UTC [3174] LOG: database system is shut down logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.602 UTC [3240] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.603 UTC [3240] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.603 UTC [3240] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.633 UTC [3240] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.640 UTC [3243] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.670 UTC [3243] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.697 UTC [3243] LOG: entering standby mode logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.723 UTC [3243] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.046 UTC [3243] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.081 UTC [3243] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.082 UTC [3240] LOG: startup process (PID 3243) exited with exit code 1 logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.082 UTC [3240] LOG: aborting startup due to startup process failure logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.082 UTC [3240] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:29 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.132 UTC [3240] LOG: database system is shut down logger.go:42: 19:39:30 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:00.098 UTC [3185] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:00.147 UTC [3185] LOG: database system is shut down logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.619 UTC [3252] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.620 UTC [3252] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.620 UTC [3252] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.645 UTC [3252] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.653 UTC [3255] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.683 UTC [3255] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.710 UTC [3255] LOG: entering standby mode logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:19.739 UTC [3255] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.061 UTC [3255] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.096 UTC [3255] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.097 UTC [3252] LOG: startup process (PID 3255) exited with exit code 1 logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.097 UTC [3252] LOG: aborting startup due to startup process failure logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.097 UTC [3252] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.148 UTC [3252] LOG: database system is shut down logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.618 UTC [3294] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.619 UTC [3294] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.619 UTC [3294] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.644 UTC [3294] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.652 UTC [3297] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.684 UTC [3297] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.711 UTC [3297] LOG: entering standby mode logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.739 UTC [3297] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.066 UTC [3297] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.101 UTC [3297] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.103 UTC [3294] LOG: startup process (PID 3297) exited with exit code 1 logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.103 UTC [3294] LOG: aborting startup due to startup process failure logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.103 UTC [3294] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:31 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.152 UTC [3294] LOG: database system is shut down logger.go:42: 19:40:02 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 18m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 18m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 18m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 16m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 16m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 14m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 14m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 14m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 17m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 17m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 17m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 16m logger.go:42: 19:40:03 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 20m logger.go:42: 19:40:05 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:40:06 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:40:07 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.082 UTC [3240] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:20.132 UTC [3240] LOG: database system is shut down logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.692 UTC [3290] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.693 UTC [3290] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.693 UTC [3290] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.718 UTC [3290] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.725 UTC [3293] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.753 UTC [3293] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.781 UTC [3293] LOG: entering standby mode logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:29.809 UTC [3293] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.131 UTC [3293] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.166 UTC [3293] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.167 UTC [3290] LOG: startup process (PID 3293) exited with exit code 1 logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.167 UTC [3290] LOG: aborting startup due to startup process failure logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.167 UTC [3290] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.237 UTC [3290] LOG: database system is shut down logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.639 UTC [3357] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.640 UTC [3357] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.640 UTC [3357] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.672 UTC [3357] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.680 UTC [3360] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.709 UTC [3360] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.736 UTC [3360] LOG: entering standby mode logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.763 UTC [3360] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.087 UTC [3360] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.121 UTC [3360] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.122 UTC [3357] LOG: startup process (PID 3360) exited with exit code 1 logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.123 UTC [3357] LOG: aborting startup due to startup process failure logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.123 UTC [3357] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:08 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.172 UTC [3357] LOG: database system is shut down logger.go:42: 19:40:09 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.103 UTC [3294] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:30.152 UTC [3294] LOG: database system is shut down logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.609 UTC [3369] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.610 UTC [3369] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.610 UTC [3369] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.636 UTC [3369] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.644 UTC [3372] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.680 UTC [3372] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.708 UTC [3372] LOG: entering standby mode logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:49.737 UTC [3372] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.060 UTC [3372] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.095 UTC [3372] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.097 UTC [3369] LOG: startup process (PID 3372) exited with exit code 1 logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.097 UTC [3369] LOG: aborting startup due to startup process failure logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.097 UTC [3369] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:50.146 UTC [3369] LOG: database system is shut down logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:59.622 UTC [3409] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:59.622 UTC [3409] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:59.622 UTC [3409] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:59.648 UTC [3409] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:59.656 UTC [3412] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:59.689 UTC [3412] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:59.715 UTC [3412] LOG: entering standby mode logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:39:59.744 UTC [3412] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:00.066 UTC [3412] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:00.102 UTC [3412] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:00.103 UTC [3409] LOG: startup process (PID 3412) exited with exit code 1 logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:00.103 UTC [3409] LOG: aborting startup due to startup process failure logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:00.103 UTC [3409] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:00.155 UTC [3409] LOG: database system is shut down logger.go:42: 19:40:41 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 18m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 18m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 19m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 16m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 17m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 15m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 15m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 15m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 18m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 18m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 18m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 17m logger.go:42: 19:40:42 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 21m logger.go:42: 19:40:43 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:40:45 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:10.108 UTC [3433] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:10.157 UTC [3433] LOG: database system is shut down logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:29.633 UTC [3502] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:29.633 UTC [3502] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:29.633 UTC [3502] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:29.667 UTC [3502] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:29.674 UTC [3505] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:29.701 UTC [3505] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:29.726 UTC [3505] LOG: entering standby mode logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:29.751 UTC [3505] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:30.073 UTC [3505] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:30.107 UTC [3505] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:30.109 UTC [3502] LOG: startup process (PID 3505) exited with exit code 1 logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:30.109 UTC [3502] LOG: aborting startup due to startup process failure logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:30.109 UTC [3502] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:30.158 UTC [3502] LOG: database system is shut down logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.613 UTC [3553] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.614 UTC [3553] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.614 UTC [3553] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.639 UTC [3553] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.646 UTC [3556] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.677 UTC [3556] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.703 UTC [3556] LOG: entering standby mode logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.730 UTC [3556] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.051 UTC [3556] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.086 UTC [3556] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.087 UTC [3553] LOG: startup process (PID 3556) exited with exit code 1 logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.087 UTC [3553] LOG: aborting startup due to startup process failure logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.087 UTC [3553] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.137 UTC [3553] LOG: database system is shut down logger.go:42: 19:40:47 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:00.103 UTC [3409] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:00.155 UTC [3409] LOG: database system is shut down logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:19.610 UTC [3484] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:19.611 UTC [3484] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:19.611 UTC [3484] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:19.636 UTC [3484] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:19.644 UTC [3487] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:19.674 UTC [3487] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:19.701 UTC [3487] LOG: entering standby mode logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:19.729 UTC [3487] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:20.052 UTC [3487] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:20.087 UTC [3487] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:20.090 UTC [3484] LOG: startup process (PID 3487) exited with exit code 1 logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:20.090 UTC [3484] LOG: aborting startup due to startup process failure logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:20.090 UTC [3484] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:20.140 UTC [3484] LOG: database system is shut down logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.604 UTC [3551] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.605 UTC [3551] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.605 UTC [3551] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.630 UTC [3551] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.638 UTC [3554] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.668 UTC [3554] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.696 UTC [3554] LOG: entering standby mode logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:39.723 UTC [3554] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.048 UTC [3554] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.082 UTC [3554] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.084 UTC [3551] LOG: startup process (PID 3554) exited with exit code 1 logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.084 UTC [3551] LOG: aborting startup due to startup process failure logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.084 UTC [3551] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:49 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:40:40.134 UTC [3551] LOG: database system is shut down logger.go:42: 19:41:20 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 19m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 19m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 20m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 17m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 17m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 16m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 15m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 16m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 19m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 19m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 19m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 18m logger.go:42: 19:41:21 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 21m logger.go:42: 19:41:22 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:41:24 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:00.140 UTC [3628] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:00.198 UTC [3628] LOG: database system is shut down logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.630 UTC [3670] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.631 UTC [3670] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.631 UTC [3670] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.657 UTC [3670] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.664 UTC [3673] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.693 UTC [3673] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.720 UTC [3673] LOG: entering standby mode logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.748 UTC [3673] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.070 UTC [3673] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.104 UTC [3673] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.106 UTC [3670] LOG: startup process (PID 3673) exited with exit code 1 logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.106 UTC [3670] LOG: aborting startup due to startup process failure logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.106 UTC [3670] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.158 UTC [3670] LOG: database system is shut down logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.637 UTC [3712] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.638 UTC [3712] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.638 UTC [3712] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.663 UTC [3712] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.672 UTC [3715] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.701 UTC [3715] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.730 UTC [3715] LOG: entering standby mode logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.757 UTC [3715] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.079 UTC [3715] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.113 UTC [3715] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.115 UTC [3712] LOG: startup process (PID 3715) exited with exit code 1 logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.115 UTC [3712] LOG: aborting startup due to startup process failure logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.115 UTC [3712] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:26 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.164 UTC [3712] LOG: database system is shut down logger.go:42: 19:41:27 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:00.097 UTC [3627] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:00.147 UTC [3627] LOG: database system is shut down logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.614 UTC [3669] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.614 UTC [3669] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.614 UTC [3669] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.640 UTC [3669] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.647 UTC [3672] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.678 UTC [3672] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.705 UTC [3672] LOG: entering standby mode logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:09.733 UTC [3672] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.059 UTC [3672] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.094 UTC [3672] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.095 UTC [3669] LOG: startup process (PID 3672) exited with exit code 1 logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.096 UTC [3669] LOG: aborting startup due to startup process failure logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.096 UTC [3669] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:10.146 UTC [3669] LOG: database system is shut down logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.630 UTC [3712] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.631 UTC [3712] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.631 UTC [3712] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.656 UTC [3712] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.667 UTC [3715] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.698 UTC [3715] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.727 UTC [3715] LOG: entering standby mode logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:19.756 UTC [3715] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.078 UTC [3715] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.113 UTC [3715] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.115 UTC [3712] LOG: startup process (PID 3715) exited with exit code 1 logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.115 UTC [3712] LOG: aborting startup due to startup process failure logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.115 UTC [3712] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:28 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:20.165 UTC [3712] LOG: database system is shut down logger.go:42: 19:41:59 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 20m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 20m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 20m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 18m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 18m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 16m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 16m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 16m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 19m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 19m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 19m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 18m logger.go:42: 19:42:00 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 22m logger.go:42: 19:42:02 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:42:03 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:42:04 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:40.097 UTC [3788] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:40.146 UTC [3788] LOG: database system is shut down logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.636 UTC [3829] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.637 UTC [3829] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.637 UTC [3829] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.662 UTC [3829] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.669 UTC [3832] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.698 UTC [3832] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.725 UTC [3832] LOG: entering standby mode logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.752 UTC [3832] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.074 UTC [3832] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.107 UTC [3832] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.109 UTC [3829] LOG: startup process (PID 3832) exited with exit code 1 logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.109 UTC [3829] LOG: aborting startup due to startup process failure logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.109 UTC [3829] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.158 UTC [3829] LOG: database system is shut down logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.629 UTC [3871] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.630 UTC [3871] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.630 UTC [3871] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.659 UTC [3871] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.668 UTC [3874] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.698 UTC [3874] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.725 UTC [3874] LOG: entering standby mode logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.754 UTC [3874] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.076 UTC [3874] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.110 UTC [3874] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.112 UTC [3871] LOG: startup process (PID 3874) exited with exit code 1 logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.112 UTC [3871] LOG: aborting startup due to startup process failure logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.112 UTC [3871] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:05 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.161 UTC [3871] LOG: database system is shut down logger.go:42: 19:42:06 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:40.091 UTC [3786] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:40.142 UTC [3786] LOG: database system is shut down logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.600 UTC [3828] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.600 UTC [3828] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.601 UTC [3828] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.626 UTC [3828] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.633 UTC [3831] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.664 UTC [3831] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.693 UTC [3831] LOG: entering standby mode logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:49.722 UTC [3831] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.045 UTC [3831] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.080 UTC [3831] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.082 UTC [3828] LOG: startup process (PID 3831) exited with exit code 1 logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.082 UTC [3828] LOG: aborting startup due to startup process failure logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.082 UTC [3828] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:50.133 UTC [3828] LOG: database system is shut down logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.626 UTC [3869] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.627 UTC [3869] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.627 UTC [3869] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.653 UTC [3869] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.661 UTC [3872] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.691 UTC [3872] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.719 UTC [3872] LOG: entering standby mode logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:41:59.748 UTC [3872] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.072 UTC [3872] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.106 UTC [3872] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.108 UTC [3869] LOG: startup process (PID 3872) exited with exit code 1 logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.108 UTC [3869] LOG: aborting startup due to startup process failure logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.108 UTC [3869] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:07 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.158 UTC [3869] LOG: database system is shut down logger.go:42: 19:42:38 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 20m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 20m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 21m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 18m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 19m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 17m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 17m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 17m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 20m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 20m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 20m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 19m logger.go:42: 19:42:39 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 23m logger.go:42: 19:42:41 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:42:42 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:42:43 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.112 UTC [3871] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.161 UTC [3871] LOG: database system is shut down logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.611 UTC [3945] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.612 UTC [3945] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.612 UTC [3945] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.637 UTC [3945] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.645 UTC [3948] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.674 UTC [3948] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.703 UTC [3948] LOG: entering standby mode logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.730 UTC [3948] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.055 UTC [3948] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.088 UTC [3948] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.090 UTC [3945] LOG: startup process (PID 3948) exited with exit code 1 logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.090 UTC [3945] LOG: aborting startup due to startup process failure logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.090 UTC [3945] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.140 UTC [3945] LOG: database system is shut down logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.603 UTC [3987] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.604 UTC [3987] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.604 UTC [3987] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.630 UTC [3987] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.642 UTC [3990] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.679 UTC [3990] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.708 UTC [3990] LOG: entering standby mode logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.735 UTC [3990] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.057 UTC [3990] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.090 UTC [3990] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.092 UTC [3987] LOG: startup process (PID 3990) exited with exit code 1 logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.092 UTC [3987] LOG: aborting startup due to startup process failure logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.092 UTC [3987] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:44 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.141 UTC [3987] LOG: database system is shut down logger.go:42: 19:42:45 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.108 UTC [3869] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:00.158 UTC [3869] LOG: database system is shut down logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.615 UTC [3944] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.615 UTC [3944] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.615 UTC [3944] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.641 UTC [3944] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.649 UTC [3947] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.685 UTC [3947] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.713 UTC [3947] LOG: entering standby mode logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:19.744 UTC [3947] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.068 UTC [3947] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.103 UTC [3947] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.105 UTC [3944] LOG: startup process (PID 3947) exited with exit code 1 logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.105 UTC [3944] LOG: aborting startup due to startup process failure logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.105 UTC [3944] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:20.157 UTC [3944] LOG: database system is shut down logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.612 UTC [3994] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.613 UTC [3994] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.613 UTC [3994] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.639 UTC [3994] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.646 UTC [3997] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.679 UTC [3997] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.707 UTC [3997] LOG: entering standby mode logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:29.736 UTC [3997] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.062 UTC [3997] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.097 UTC [3997] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.098 UTC [3994] LOG: startup process (PID 3997) exited with exit code 1 logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.098 UTC [3994] LOG: aborting startup due to startup process failure logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.098 UTC [3994] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:46 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.149 UTC [3994] LOG: database system is shut down logger.go:42: 19:43:17 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 21m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 21m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 22m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 19m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 19m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 18m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 17m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 17m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 21m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 21m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 21m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 20m logger.go:42: 19:43:18 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 23m logger.go:42: 19:43:20 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:43:21 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:43:22 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.092 UTC [3987] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.141 UTC [3987] LOG: database system is shut down logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.628 UTC [4062] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.629 UTC [4062] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.629 UTC [4062] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.655 UTC [4062] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.662 UTC [4065] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.691 UTC [4065] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.727 UTC [4065] LOG: entering standby mode logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.753 UTC [4065] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.075 UTC [4065] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.110 UTC [4065] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.112 UTC [4062] LOG: startup process (PID 4065) exited with exit code 1 logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.112 UTC [4062] LOG: aborting startup due to startup process failure logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.112 UTC [4062] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.161 UTC [4062] LOG: database system is shut down logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.619 UTC [4129] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.620 UTC [4129] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.620 UTC [4129] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.645 UTC [4129] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.652 UTC [4132] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.682 UTC [4132] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.709 UTC [4132] LOG: entering standby mode logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.737 UTC [4132] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.059 UTC [4132] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.093 UTC [4132] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.095 UTC [4129] LOG: startup process (PID 4132) exited with exit code 1 logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.095 UTC [4129] LOG: aborting startup due to startup process failure logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.095 UTC [4129] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:23 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.144 UTC [4129] LOG: database system is shut down logger.go:42: 19:43:24 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.098 UTC [3994] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:30.149 UTC [3994] LOG: database system is shut down logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.620 UTC [4069] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.621 UTC [4069] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.621 UTC [4069] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.646 UTC [4069] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.655 UTC [4072] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.685 UTC [4072] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.716 UTC [4072] LOG: entering standby mode logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:49.745 UTC [4072] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.070 UTC [4072] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.106 UTC [4072] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.107 UTC [4069] LOG: startup process (PID 4072) exited with exit code 1 logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.107 UTC [4069] LOG: aborting startup due to startup process failure logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.107 UTC [4069] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:42:50.157 UTC [4069] LOG: database system is shut down logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.614 UTC [4135] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.614 UTC [4135] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.614 UTC [4135] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.640 UTC [4135] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.648 UTC [4138] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.681 UTC [4138] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.709 UTC [4138] LOG: entering standby mode logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:09.738 UTC [4138] LOG: restored log file "00000003.history" from archive logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.063 UTC [4138] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.098 UTC [4138] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.100 UTC [4135] LOG: startup process (PID 4138) exited with exit code 1 logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.100 UTC [4135] LOG: aborting startup due to startup process failure logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.100 UTC [4135] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:43:25 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.151 UTC [4135] LOG: database system is shut down logger.go:42: 19:43:56 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 22m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 22m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 22m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 20m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 20m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 18m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 18m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 18m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 21m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 21m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 21m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 20m logger.go:42: 19:43:57 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 24m logger.go:42: 19:43:59 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:44:00 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:44:01 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.095 UTC [4129] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.144 UTC [4129] LOG: database system is shut down logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.622 UTC [4213] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.622 UTC [4213] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.622 UTC [4213] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.648 UTC [4213] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.655 UTC [4216] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.687 UTC [4216] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.715 UTC [4216] LOG: entering standby mode logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.742 UTC [4216] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.065 UTC [4216] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.099 UTC [4216] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.101 UTC [4213] LOG: startup process (PID 4216) exited with exit code 1 logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.101 UTC [4213] LOG: aborting startup due to startup process failure logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.101 UTC [4213] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.152 UTC [4213] LOG: database system is shut down logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.609 UTC [4280] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.610 UTC [4280] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.610 UTC [4280] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.635 UTC [4280] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.642 UTC [4283] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.673 UTC [4283] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.700 UTC [4283] LOG: entering standby mode logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.727 UTC [4283] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.048 UTC [4283] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.083 UTC [4283] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.085 UTC [4280] LOG: startup process (PID 4283) exited with exit code 1 logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.085 UTC [4280] LOG: aborting startup due to startup process failure logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.085 UTC [4280] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:02 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.135 UTC [4280] LOG: database system is shut down logger.go:42: 19:44:03 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.100 UTC [4135] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:10.151 UTC [4135] LOG: database system is shut down logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.625 UTC [4210] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.625 UTC [4210] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.625 UTC [4210] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.651 UTC [4210] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.659 UTC [4213] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.691 UTC [4213] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.720 UTC [4213] LOG: entering standby mode logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:29.747 UTC [4213] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.070 UTC [4213] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.105 UTC [4213] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.106 UTC [4210] LOG: startup process (PID 4213) exited with exit code 1 logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.106 UTC [4210] LOG: aborting startup due to startup process failure logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.106 UTC [4210] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:30.159 UTC [4210] LOG: database system is shut down logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.609 UTC [4277] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.610 UTC [4277] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.610 UTC [4277] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.635 UTC [4277] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.643 UTC [4280] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.674 UTC [4280] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.701 UTC [4280] LOG: entering standby mode logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:49.730 UTC [4280] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.053 UTC [4280] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.089 UTC [4280] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.091 UTC [4277] LOG: startup process (PID 4280) exited with exit code 1 logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.091 UTC [4277] LOG: aborting startup due to startup process failure logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.091 UTC [4277] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:04 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.141 UTC [4277] LOG: database system is shut down logger.go:42: 19:44:35 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 22m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 22m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 23m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 20m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 20m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 19m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 19m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 19m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 22m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 22m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 22m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 21m logger.go:42: 19:44:36 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 25m logger.go:42: 19:44:38 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:44:39 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:44:40 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.085 UTC [4280] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.135 UTC [4280] LOG: database system is shut down logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.619 UTC [4355] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.620 UTC [4355] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.620 UTC [4355] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.645 UTC [4355] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.653 UTC [4358] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.683 UTC [4358] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.712 UTC [4358] LOG: entering standby mode logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.740 UTC [4358] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.064 UTC [4358] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.099 UTC [4358] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.101 UTC [4355] LOG: startup process (PID 4358) exited with exit code 1 logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.101 UTC [4355] LOG: aborting startup due to startup process failure logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.101 UTC [4355] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.151 UTC [4355] LOG: database system is shut down logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.600 UTC [4421] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.601 UTC [4421] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.601 UTC [4421] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.629 UTC [4421] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.636 UTC [4424] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.664 UTC [4424] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.690 UTC [4424] LOG: entering standby mode logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.716 UTC [4424] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.037 UTC [4424] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.071 UTC [4424] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.073 UTC [4421] LOG: startup process (PID 4424) exited with exit code 1 logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.073 UTC [4421] LOG: aborting startup due to startup process failure logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.073 UTC [4421] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:41 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.124 UTC [4421] LOG: database system is shut down logger.go:42: 19:44:42 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.091 UTC [4277] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:43:50.141 UTC [4277] LOG: database system is shut down logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.630 UTC [4353] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.630 UTC [4353] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.630 UTC [4353] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.656 UTC [4353] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.664 UTC [4356] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.693 UTC [4356] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.720 UTC [4356] LOG: entering standby mode logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:09.749 UTC [4356] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.074 UTC [4356] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.108 UTC [4356] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.110 UTC [4353] LOG: startup process (PID 4356) exited with exit code 1 logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.110 UTC [4353] LOG: aborting startup due to startup process failure logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.110 UTC [4353] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:10.161 UTC [4353] LOG: database system is shut down logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.618 UTC [4421] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.619 UTC [4421] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.619 UTC [4421] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.645 UTC [4421] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.653 UTC [4424] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.683 UTC [4424] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.710 UTC [4424] LOG: entering standby mode logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:29.736 UTC [4424] LOG: restored log file "00000003.history" from archive logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.059 UTC [4424] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.095 UTC [4424] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.096 UTC [4421] LOG: startup process (PID 4424) exited with exit code 1 logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.096 UTC [4421] LOG: aborting startup due to startup process failure logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.096 UTC [4421] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:44:43 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:30.146 UTC [4421] LOG: database system is shut down logger.go:42: 19:45:14 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 23m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 23m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 24m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 21m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 21m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 20m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 19m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 19m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 23m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 23m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 23m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 22m logger.go:42: 19:45:15 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 25m logger.go:42: 19:45:17 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:45:18 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:45:19 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:50.079 UTC [4495] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:50.136 UTC [4495] LOG: database system is shut down logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.608 UTC [4537] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.608 UTC [4537] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.609 UTC [4537] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.643 UTC [4537] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.652 UTC [4540] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.684 UTC [4540] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.712 UTC [4540] LOG: entering standby mode logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.738 UTC [4540] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.061 UTC [4540] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.095 UTC [4540] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.097 UTC [4537] LOG: startup process (PID 4540) exited with exit code 1 logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.097 UTC [4537] LOG: aborting startup due to startup process failure logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.097 UTC [4537] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.146 UTC [4537] LOG: database system is shut down logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.593 UTC [4604] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.594 UTC [4604] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.594 UTC [4604] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.619 UTC [4604] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.627 UTC [4607] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.655 UTC [4607] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.684 UTC [4607] LOG: entering standby mode logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.712 UTC [4607] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.035 UTC [4607] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.070 UTC [4607] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.072 UTC [4604] LOG: startup process (PID 4607) exited with exit code 1 logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.072 UTC [4604] LOG: aborting startup due to startup process failure logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.072 UTC [4604] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:20 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.120 UTC [4604] LOG: database system is shut down logger.go:42: 19:45:21 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:50.114 UTC [4497] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:50.165 UTC [4497] LOG: database system is shut down logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.634 UTC [4539] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.634 UTC [4539] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.634 UTC [4539] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.661 UTC [4539] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.670 UTC [4542] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.702 UTC [4542] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.733 UTC [4542] LOG: entering standby mode logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:44:59.765 UTC [4542] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.094 UTC [4542] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.129 UTC [4542] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.130 UTC [4539] LOG: startup process (PID 4542) exited with exit code 1 logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.130 UTC [4539] LOG: aborting startup due to startup process failure logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.130 UTC [4539] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.181 UTC [4539] LOG: database system is shut down logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.644 UTC [4606] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.645 UTC [4606] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.645 UTC [4606] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.670 UTC [4606] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.678 UTC [4609] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.708 UTC [4609] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.736 UTC [4609] LOG: entering standby mode logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.766 UTC [4609] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.094 UTC [4609] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.129 UTC [4609] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.131 UTC [4606] LOG: startup process (PID 4609) exited with exit code 1 logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.131 UTC [4606] LOG: aborting startup due to startup process failure logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.131 UTC [4606] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:22 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.183 UTC [4606] LOG: database system is shut down logger.go:42: 19:45:53 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 24m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 24m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 24m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 21m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 22m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 20m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 20m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 20m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 23m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 23m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 23m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 22m logger.go:42: 19:45:54 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 26m logger.go:42: 19:45:56 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:45:57 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:45:58 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.097 UTC [4537] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:00.146 UTC [4537] LOG: database system is shut down logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.593 UTC [4604] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.594 UTC [4604] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.594 UTC [4604] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.619 UTC [4604] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.627 UTC [4607] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.655 UTC [4607] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.684 UTC [4607] LOG: entering standby mode logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:19.712 UTC [4607] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.035 UTC [4607] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.070 UTC [4607] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.072 UTC [4604] LOG: startup process (PID 4607) exited with exit code 1 logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.072 UTC [4604] LOG: aborting startup due to startup process failure logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.072 UTC [4604] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:20.120 UTC [4604] LOG: database system is shut down logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:39.593 UTC [4679] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:39.593 UTC [4679] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:39.593 UTC [4679] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:39.619 UTC [4679] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:39.627 UTC [4682] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:39.655 UTC [4682] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:39.683 UTC [4682] LOG: entering standby mode logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:39.710 UTC [4682] LOG: restored log file "00000003.history" from archive logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:40.032 UTC [4682] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:40.065 UTC [4682] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:40.067 UTC [4679] LOG: startup process (PID 4682) exited with exit code 1 logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:40.067 UTC [4679] LOG: aborting startup due to startup process failure logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:40.067 UTC [4679] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:45:59 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:40.115 UTC [4679] LOG: database system is shut down logger.go:42: 19:46:00 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:30.125 UTC [4655] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:30.178 UTC [4655] LOG: database system is shut down logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:49.626 UTC [4723] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:49.627 UTC [4723] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:49.627 UTC [4723] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:49.652 UTC [4723] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:49.660 UTC [4726] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:49.695 UTC [4726] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:49.722 UTC [4726] LOG: entering standby mode logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:49.752 UTC [4726] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:50.076 UTC [4726] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:50.111 UTC [4726] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:50.112 UTC [4723] LOG: startup process (PID 4726) exited with exit code 1 logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:50.112 UTC [4723] LOG: aborting startup due to startup process failure logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:50.112 UTC [4723] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:50.165 UTC [4723] LOG: database system is shut down logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.645 UTC [4765] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.646 UTC [4765] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.646 UTC [4765] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.672 UTC [4765] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.680 UTC [4768] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.712 UTC [4768] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.745 UTC [4768] LOG: entering standby mode logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.776 UTC [4768] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.110 UTC [4768] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.145 UTC [4768] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.146 UTC [4765] LOG: startup process (PID 4768) exited with exit code 1 logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.146 UTC [4765] LOG: aborting startup due to startup process failure logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.146 UTC [4765] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:01 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.198 UTC [4765] LOG: database system is shut down logger.go:42: 19:46:32 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 24m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 24m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 25m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 22m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 22m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 21m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 21m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 21m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 24m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 24m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 24m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 23m logger.go:42: 19:46:33 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 27m logger.go:42: 19:46:35 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:46:36 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:46:37 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:40.067 UTC [4679] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:40.115 UTC [4679] LOG: database system is shut down logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.628 UTC [4754] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.629 UTC [4754] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.629 UTC [4754] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.654 UTC [4754] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.662 UTC [4757] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.690 UTC [4757] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.716 UTC [4757] LOG: entering standby mode logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:45:59.742 UTC [4757] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.063 UTC [4757] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.098 UTC [4757] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.099 UTC [4754] LOG: startup process (PID 4757) exited with exit code 1 logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.099 UTC [4754] LOG: aborting startup due to startup process failure logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.099 UTC [4754] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.147 UTC [4754] LOG: database system is shut down logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.610 UTC [4821] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.611 UTC [4821] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.611 UTC [4821] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.642 UTC [4821] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.650 UTC [4824] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.678 UTC [4824] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.706 UTC [4824] LOG: entering standby mode logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.732 UTC [4824] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.054 UTC [4824] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.088 UTC [4824] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.090 UTC [4821] LOG: startup process (PID 4824) exited with exit code 1 logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.090 UTC [4821] LOG: aborting startup due to startup process failure logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.090 UTC [4821] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:38 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.139 UTC [4821] LOG: database system is shut down logger.go:42: 19:46:39 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.146 UTC [4765] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:00.198 UTC [4765] LOG: database system is shut down logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.627 UTC [4849] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.628 UTC [4849] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.628 UTC [4849] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.654 UTC [4849] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.662 UTC [4852] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.691 UTC [4852] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.719 UTC [4852] LOG: entering standby mode logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:19.747 UTC [4852] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.073 UTC [4852] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.108 UTC [4852] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.110 UTC [4849] LOG: startup process (PID 4852) exited with exit code 1 logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.110 UTC [4849] LOG: aborting startup due to startup process failure logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.110 UTC [4849] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:20.163 UTC [4849] LOG: database system is shut down logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:39.630 UTC [4915] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:39.631 UTC [4915] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:39.631 UTC [4915] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:39.657 UTC [4915] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:39.665 UTC [4918] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:39.695 UTC [4918] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:39.723 UTC [4918] LOG: entering standby mode logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:39.751 UTC [4918] LOG: restored log file "00000003.history" from archive logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.072 UTC [4918] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.108 UTC [4918] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.110 UTC [4915] LOG: startup process (PID 4918) exited with exit code 1 logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.110 UTC [4915] LOG: aborting startup due to startup process failure logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.110 UTC [4915] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:46:40 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.160 UTC [4915] LOG: database system is shut down logger.go:42: 19:47:11 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 25m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 25m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 26m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 23m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 23m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 22m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 21m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 21m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 25m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 25m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 25m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 24m logger.go:42: 19:47:12 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 27m logger.go:42: 19:47:14 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:47:15 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:47:16 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.073 UTC [4896] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.123 UTC [4896] LOG: database system is shut down logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.598 UTC [4938] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.599 UTC [4938] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.599 UTC [4938] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.624 UTC [4938] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.631 UTC [4941] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.660 UTC [4941] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.690 UTC [4941] LOG: entering standby mode logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.716 UTC [4941] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.038 UTC [4941] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.072 UTC [4941] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.074 UTC [4938] LOG: startup process (PID 4941) exited with exit code 1 logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.074 UTC [4938] LOG: aborting startup due to startup process failure logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.074 UTC [4938] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.121 UTC [4938] LOG: database system is shut down logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.605 UTC [5004] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.605 UTC [5004] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.606 UTC [5004] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.630 UTC [5004] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.638 UTC [5007] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.668 UTC [5007] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.696 UTC [5007] LOG: entering standby mode logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.722 UTC [5007] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.045 UTC [5007] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.080 UTC [5007] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.082 UTC [5004] LOG: startup process (PID 5007) exited with exit code 1 logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.082 UTC [5004] LOG: aborting startup due to startup process failure logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.082 UTC [5004] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:17 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.130 UTC [5004] LOG: database system is shut down logger.go:42: 19:47:18 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.110 UTC [4915] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:40.160 UTC [4915] LOG: database system is shut down logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.627 UTC [4964] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.628 UTC [4964] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.628 UTC [4964] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.653 UTC [4964] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.661 UTC [4967] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.692 UTC [4967] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.719 UTC [4967] LOG: entering standby mode logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:49.746 UTC [4967] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.071 UTC [4967] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.106 UTC [4967] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.108 UTC [4964] LOG: startup process (PID 4967) exited with exit code 1 logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.108 UTC [4964] LOG: aborting startup due to startup process failure logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.108 UTC [4964] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:46:50.157 UTC [4964] LOG: database system is shut down logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.604 UTC [5032] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.605 UTC [5032] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.605 UTC [5032] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.630 UTC [5032] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.639 UTC [5035] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.674 UTC [5035] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.701 UTC [5035] LOG: entering standby mode logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:09.729 UTC [5035] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.053 UTC [5035] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.087 UTC [5035] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.089 UTC [5032] LOG: startup process (PID 5035) exited with exit code 1 logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.089 UTC [5032] LOG: aborting startup due to startup process failure logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.089 UTC [5032] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:19 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.138 UTC [5032] LOG: database system is shut down logger.go:42: 19:47:50 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 26m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 26m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 26m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 23m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 24m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 22m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 22m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 22m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 25m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 25m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 25m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 24m logger.go:42: 19:47:51 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 28m logger.go:42: 19:47:53 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:47:54 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:47:55 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.082 UTC [5004] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.130 UTC [5004] LOG: database system is shut down logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.612 UTC [5079] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.612 UTC [5079] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.612 UTC [5079] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.638 UTC [5079] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.646 UTC [5082] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.675 UTC [5082] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.702 UTC [5082] LOG: entering standby mode logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.727 UTC [5082] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.050 UTC [5082] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.084 UTC [5082] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.085 UTC [5079] LOG: startup process (PID 5082) exited with exit code 1 logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.085 UTC [5079] LOG: aborting startup due to startup process failure logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.085 UTC [5079] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.135 UTC [5079] LOG: database system is shut down logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.730 UTC [5146] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.731 UTC [5146] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.731 UTC [5146] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.758 UTC [5146] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.768 UTC [5149] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.799 UTC [5149] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.830 UTC [5149] LOG: entering standby mode logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.865 UTC [5149] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.192 UTC [5149] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.231 UTC [5149] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.232 UTC [5146] LOG: startup process (PID 5149) exited with exit code 1 logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.233 UTC [5146] LOG: aborting startup due to startup process failure logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.233 UTC [5146] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:56 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.296 UTC [5146] LOG: database system is shut down logger.go:42: 19:47:57 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.089 UTC [5032] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:10.138 UTC [5032] LOG: database system is shut down logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.605 UTC [5118] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.606 UTC [5118] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.606 UTC [5118] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.632 UTC [5118] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.640 UTC [5121] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.671 UTC [5121] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.702 UTC [5121] LOG: entering standby mode logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:29.742 UTC [5121] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.065 UTC [5121] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.100 UTC [5121] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.101 UTC [5118] LOG: startup process (PID 5121) exited with exit code 1 logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.101 UTC [5118] LOG: aborting startup due to startup process failure logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.101 UTC [5118] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:30.153 UTC [5118] LOG: database system is shut down logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.605 UTC [5186] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.606 UTC [5186] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.606 UTC [5186] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.631 UTC [5186] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.638 UTC [5189] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.670 UTC [5189] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.700 UTC [5189] LOG: entering standby mode logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:49.730 UTC [5189] LOG: restored log file "00000003.history" from archive logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.053 UTC [5189] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.088 UTC [5189] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.089 UTC [5186] LOG: startup process (PID 5189) exited with exit code 1 logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.089 UTC [5186] LOG: aborting startup due to startup process failure logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.089 UTC [5186] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:47:58 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.141 UTC [5186] LOG: database system is shut down logger.go:42: 19:48:29 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 26m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 26m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 27m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 24m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 24m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 23m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 23m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 23m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 26m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 26m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 26m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 25m logger.go:42: 19:48:30 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 28m logger.go:42: 19:48:32 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:48:33 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:48:34 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.233 UTC [5146] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.296 UTC [5146] LOG: database system is shut down logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.810 UTC [5219] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.810 UTC [5219] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.810 UTC [5219] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.839 UTC [5219] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.849 UTC [5222] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.888 UTC [5222] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.922 UTC [5222] LOG: entering standby mode logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.955 UTC [5222] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.299 UTC [5222] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.340 UTC [5222] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.342 UTC [5219] LOG: startup process (PID 5222) exited with exit code 1 logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.342 UTC [5219] LOG: aborting startup due to startup process failure logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.342 UTC [5219] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.404 UTC [5219] LOG: database system is shut down logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.972 UTC [5286] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.973 UTC [5286] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.973 UTC [5286] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.003 UTC [5286] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.015 UTC [5289] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.055 UTC [5289] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.090 UTC [5289] LOG: entering standby mode logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.128 UTC [5289] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.458 UTC [5289] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.500 UTC [5289] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.502 UTC [5286] LOG: startup process (PID 5289) exited with exit code 1 logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.502 UTC [5286] LOG: aborting startup due to startup process failure logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.503 UTC [5286] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:35 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.564 UTC [5286] LOG: database system is shut down logger.go:42: 19:48:36 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.089 UTC [5186] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:47:50.141 UTC [5186] LOG: database system is shut down logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.623 UTC [5261] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.623 UTC [5261] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.624 UTC [5261] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.649 UTC [5261] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.656 UTC [5264] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.691 UTC [5264] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.723 UTC [5264] LOG: entering standby mode logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:09.753 UTC [5264] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.075 UTC [5264] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.110 UTC [5264] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.111 UTC [5261] LOG: startup process (PID 5264) exited with exit code 1 logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.111 UTC [5261] LOG: aborting startup due to startup process failure logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.111 UTC [5261] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:10.161 UTC [5261] LOG: database system is shut down logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.618 UTC [5327] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.619 UTC [5327] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.619 UTC [5327] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.644 UTC [5327] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.652 UTC [5330] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.696 UTC [5330] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.730 UTC [5330] LOG: entering standby mode logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:29.766 UTC [5330] LOG: restored log file "00000003.history" from archive logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.090 UTC [5330] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.124 UTC [5330] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.126 UTC [5327] LOG: startup process (PID 5330) exited with exit code 1 logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.126 UTC [5327] LOG: aborting startup due to startup process failure logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.126 UTC [5327] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:48:37 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:30.178 UTC [5327] LOG: database system is shut down logger.go:42: 19:49:08 | major-upgrade-13-to-14/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w 0/1 Completed 0 27m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz 0/1 Completed 0 27m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | 13-to-14-pgdata-klq9k 0/1 Completed 0 28m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-96kr-qw2pf 0/1 Completed 0 25m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-backup-gl5p-4zd6n 0/1 Completed 0 25m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-6vhh-0 4/4 Running 0 24m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-dr44-0 3/4 Running 0 23m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-instance1-zwr5-0 3/4 Running 0 23m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz 2/2 Running 0 27m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 2/2 Running 0 27m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz 2/2 Running 0 27m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | major-upgrade-13-to-14-repo-host-0 2/2 Running 0 26m logger.go:42: 19:49:09 | major-upgrade-13-to-14/10-run-restore | pg-client-65d98588cc-cc9lq 1/1 Running 0 29m logger.go:42: 19:49:11 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-6vhh-0: logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.436 UTC [134] LOG: redo is not required logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.558 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.584 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.585 UTC [134] LOG: selected new timeline ID: 2 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.635 UTC [134] LOG: archive recovery complete logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '14' and system-id '7609029758364954718' logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.659 P00 ERROR: [103]: unable to find a valid repository logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.685 UTC [131] LOG: database system is ready to accept connections logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.702 UTC [168] LOG: pg_cron scheduler started logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.801 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:22:26.923 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 14, system-id 7609029758364954718 do not match repo1 stanza version 13, system-id 7609029347188273282 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000C.00000028.backup": read 338 of 8192 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000002000000000000000E.00000028.backup": read 338 of 8192 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "000000020000000000000010.00000028.backup": read 374 of 8192 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.879 UTC [124] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.880 UTC [124] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.905 UTC [124] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.919 UTC [127] LOG: database system was shut down at 2026-02-20 19:24:59 UTC logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:16.943 UTC [124] LOG: database system is ready to accept connections logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.003 UTC [134] LOG: pg_cron scheduler started logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | pg_waldump: fatal: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:49:12 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:25:17.801 UTC [124] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:49:13 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-dr44-0: logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:40.560 UTC [5336] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:40.624 UTC [5336] LOG: database system is shut down logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.932 UTC [5402] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.933 UTC [5402] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.933 UTC [5402] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.964 UTC [5402] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.983 UTC [5405] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.034 UTC [5405] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.066 UTC [5405] LOG: entering standby mode logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.109 UTC [5405] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.446 UTC [5405] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.487 UTC [5405] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.488 UTC [5402] LOG: startup process (PID 5405) exited with exit code 1 logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.489 UTC [5402] LOG: aborting startup due to startup process failure logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.489 UTC [5402] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.555 UTC [5402] LOG: database system is shut down logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.937 UTC [5444] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.939 UTC [5444] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.939 UTC [5444] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.967 UTC [5444] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.982 UTC [5447] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.027 UTC [5447] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.067 UTC [5447] LOG: entering standby mode logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.104 UTC [5447] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.435 UTC [5447] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.476 UTC [5447] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.478 UTC [5444] LOG: startup process (PID 5447) exited with exit code 1 logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.478 UTC [5444] LOG: aborting startup due to startup process failure logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.478 UTC [5444] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:14 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.538 UTC [5444] LOG: database system is shut down logger.go:42: 19:49:15 | major-upgrade-13-to-14/10-run-restore | PostgreSQL logs from major-upgrade-13-to-14-instance1-zwr5-0: logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:40.128 UTC [5377] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:40.180 UTC [5377] LOG: database system is shut down logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.623 UTC [5444] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.624 UTC [5444] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.624 UTC [5444] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.650 UTC [5444] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.658 UTC [5447] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.699 UTC [5447] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.735 UTC [5447] LOG: entering standby mode logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:48:59.764 UTC [5447] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.091 UTC [5447] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.130 UTC [5447] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.132 UTC [5444] LOG: startup process (PID 5447) exited with exit code 1 logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.132 UTC [5444] LOG: aborting startup due to startup process failure logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.132 UTC [5444] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:00.184 UTC [5444] LOG: database system is shut down logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.627 UTC [5486] LOG: starting PostgreSQL 14.20 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.627 UTC [5486] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.628 UTC [5486] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.654 UTC [5486] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.665 UTC [5489] LOG: database system was shut down in recovery at 2026-02-20 19:24:34 UTC logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.705 UTC [5489] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.738 UTC [5489] LOG: entering standby mode logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:09.770 UTC [5489] LOG: restored log file "00000003.history" from archive logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.097 UTC [5489] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.133 UTC [5489] FATAL: requested timeline 3 does not contain minimum recovery point 0/13005A88 on timeline 2 logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.135 UTC [5486] LOG: startup process (PID 5489) exited with exit code 1 logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.135 UTC [5486] LOG: aborting startup due to startup process failure logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.135 UTC [5486] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:49:16 | major-upgrade-13-to-14/10-run-restore | 2026-02-20 19:49:10.196 UTC [5486] LOG: database system is shut down logger.go:42: 19:49:32 | major-upgrade-13-to-14/10-run-restore | test step failed 10-run-restore logger.go:42: 19:49:32 | major-upgrade-13-to-14/10-run-restore | collecting log output for [type==pod,label: postgres-operator.crunchydata.com/data=postgres] logger.go:42: 19:49:32 | major-upgrade-13-to-14/10-run-restore | running command: [kubectl logs --prefix -l postgres-operator.crunchydata.com/data=postgres -n kuttl-test-patient-primate --all-containers --tail=30] logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + IFS=, logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + read -ra extensions logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/14') logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + declare -a args logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + [[ -n '' ]] logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + [[ '' == \t\r\u\e ]] logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + [[ '' == \t\r\u\e ]] logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] ++ basename -- /pgdata/extension/14/pg_cron-pg14-1.6.6.installed logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + filename=pg_cron-pg14-1.6.6.installed logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + key=pg_cron-pg14-1.6.6 logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + [[ pg_cron-pg14-1.6.6 == \* ]] logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + [[ ! pg_cron-pg14-1.6.6 =~ pg_cron-pg14-1.6.6 ]] logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + for key in "${extensions[@]}" logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + '[' -f /pgdata/extension/14/pg_cron-pg14-1.6.6.installed ']' logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + echo 'Extension pg_cron-pg14-1.6.6 already installed' logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] + continue logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/extension-installer-14] Extension pg_cron-pg14-1.6.6 already installed logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -u logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -g logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -u logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -g logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -u logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -g logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -u logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -g logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -u logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -g logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -u logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] ++ id -g logger.go:42: 19:49:33 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + IFS=, logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + read -ra extensions logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/14') logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + declare -a args logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + [[ -n '' ]] logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + [[ '' == \t\r\u\e ]] logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + [[ '' == \t\r\u\e ]] logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] ++ basename -- /pgdata/extension/14/pg_cron-pg14-1.6.6.installed logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + filename=pg_cron-pg14-1.6.6.installed logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + key=pg_cron-pg14-1.6.6 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + [[ pg_cron-pg14-1.6.6 == \* ]] logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + [[ ! pg_cron-pg14-1.6.6 =~ pg_cron-pg14-1.6.6 ]] logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] Extension pg_cron-pg14-1.6.6 already installed logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + for key in "${extensions[@]}" logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + '[' -f /pgdata/extension/14/pg_cron-pg14-1.6.6.installed ']' logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + echo 'Extension pg_cron-pg14-1.6.6 already installed' logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/extension-installer-14] + continue logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] wal_level setting: logical logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] wal_log_hints setting: on logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] max_connections setting: 100 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] max_worker_processes setting: 8 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] max_wal_senders setting: 10 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] max_prepared_xacts setting: 0 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] max_locks_per_xact setting: 64 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] track_commit_timestamp setting: on logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Maximum data alignment: 8 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Database block size: 8192 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Blocks per segment of large relation: 131072 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] WAL block size: 8192 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Bytes per WAL segment: 16777216 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Maximum length of identifiers: 64 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Maximum columns in an index: 32 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Maximum size of a TOAST chunk: 1996 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Size of a large-object chunk: 2048 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Date/time type storage: 64-bit integers logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Float8 argument passing: by value logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Data page checksum version: 1 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] Mock authentication nonce: 51e0f785d39259f2c47d33f0b19991494200af84ed70e6b434e1e4f71bc9cfc2 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] 2026-02-20 19:49:28,843 INFO: Lock owner: major-upgrade-13-to-14-instance1-6vhh-0; I am major-upgrade-13-to-14-instance1-zwr5-0 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] 2026-02-20 19:49:28,844 INFO: starting as a secondary logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] 2026-02-20 19:49:29.359 UTC [5561] LOG: pgaudit extension initialized logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] 2026-02-20 19:49:29,375 INFO: postmaster pid=5561 logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] /tmp/postgres:5432 - no response logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] 2026-02-20 19:49:29.619 UTC [5561] LOG: redirecting log output to logging collector process logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] 2026-02-20 19:49:29.619 UTC [5561] HINT: Future log output will appear in directory "log". logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/database] 2026-02-20 19:49:30,389 ERROR: postmaster is not running logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:44:48,823 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:44:58,825 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:45:08,826 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:45:18,823 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:45:28,826 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:45:38,822 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:45:48,821 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:45:58,826 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:46:08,824 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:46:18,823 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:46:18,988 ERROR: ObjectCache.run ProtocolError("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:46:28,821 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:46:38,824 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:46:48,831 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:46:58,824 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:47:08,825 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:47:18,823 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:47:28,827 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:47:38,822 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:47:48,828 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:47:58,823 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:48:08,823 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:48:18,822 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:48:28,822 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:48:38,826 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:48:48,824 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:48:58,822 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:49:08,824 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:49:18,823 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database] 2026-02-20 19:49:28,827 INFO: no action. I am (major-upgrade-13-to-14-instance1-6vhh-0), the leader with the lock logger.go:42: 19:49:34 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=105-4df4c66b --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@5de6e25e-169f-4d1f-bae2-62524e66d2e5=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -u logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -g logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -u logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -g logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -u logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -g logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -u logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -g logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -u logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -g logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -u logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] ++ id -g logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=113-dcb15a70 --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@5de6e25e-169f-4d1f-bae2-62524e66d2e5=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + IFS=, logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + read -ra extensions logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/14') logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + declare -a args logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + [[ -n '' ]] logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + [[ '' == \t\r\u\e ]] logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + [[ '' == \t\r\u\e ]] logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] ++ basename -- /pgdata/extension/14/pg_cron-pg14-1.6.6.installed logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + filename=pg_cron-pg14-1.6.6.installed logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + key=pg_cron-pg14-1.6.6 logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + [[ pg_cron-pg14-1.6.6 == \* ]] logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + [[ ! pg_cron-pg14-1.6.6 =~ pg_cron-pg14-1.6.6 ]] logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + for key in "${extensions[@]}" logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + '[' -f /pgdata/extension/14/pg_cron-pg14-1.6.6.installed ']' logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + echo 'Extension pg_cron-pg14-1.6.6 already installed' logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] + continue logger.go:42: 19:49:35 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/extension-installer-14] Extension pg_cron-pg14-1.6.6 already installed logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] Initializing ... logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-14/bin/postgres logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 14.20 - Percona Distribution logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg14 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg14 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: data version::14 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg14_wal logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] Initializing ... logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-14/bin/postgres logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 14.20 - Percona Distribution logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg14 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg14 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: bootstrap directory::/pgdata/pg14_bootstrap logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: data version::14 logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg14_wal logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-6vhh-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -u logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -g logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -u logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -g logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -u logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -g logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -u logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -g logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -u logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -g logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -u logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] ++ id -g logger.go:42: 19:49:36 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] Initializing ... logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-14/bin/postgres logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 14.20 - Percona Distribution logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg14 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg14 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: data version::14 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg14_wal logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] wal_level setting: logical logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] wal_log_hints setting: on logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] max_connections setting: 100 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] max_worker_processes setting: 8 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] max_wal_senders setting: 10 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] max_prepared_xacts setting: 0 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] max_locks_per_xact setting: 64 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] track_commit_timestamp setting: on logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Maximum data alignment: 8 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Database block size: 8192 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Blocks per segment of large relation: 131072 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] WAL block size: 8192 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Bytes per WAL segment: 16777216 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Maximum length of identifiers: 64 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Maximum columns in an index: 32 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Maximum size of a TOAST chunk: 1996 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Size of a large-object chunk: 2048 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Date/time type storage: 64-bit integers logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Float8 argument passing: by value logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Data page checksum version: 1 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] Mock authentication nonce: 51e0f785d39259f2c47d33f0b19991494200af84ed70e6b434e1e4f71bc9cfc2 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] 2026-02-20 19:49:28,846 INFO: Lock owner: major-upgrade-13-to-14-instance1-6vhh-0; I am major-upgrade-13-to-14-instance1-dr44-0 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] 2026-02-20 19:49:28,848 INFO: starting as a secondary logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] 2026-02-20 19:49:29.551 UTC [5519] LOG: pgaudit extension initialized logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] 2026-02-20 19:49:29,585 INFO: postmaster pid=5519 logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] /tmp/postgres:5432 - no response logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] 2026-02-20 19:49:29.866 UTC [5519] LOG: redirecting log output to logging collector process logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] 2026-02-20 19:49:29.866 UTC [5519] HINT: Future log output will appear in directory "log". logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-dr44-0/database] 2026-02-20 19:49:30,604 ERROR: postmaster is not running logger.go:42: 19:49:37 | major-upgrade-13-to-14/10-run-restore | [pod/major-upgrade-13-to-14-instance1-zwr5-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=108-4ed7d96a --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@5de6e25e-169f-4d1f-bae2-62524e66d2e5=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key case.go:328: failed in step 10-run-restore case.go:330: --- PerconaPGCluster:kuttl-test-patient-primate/major-upgrade-13-to-14 +++ PerconaPGCluster:kuttl-test-patient-primate/major-upgrade-13-to-14 @@ -1,18 +1,41 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"pgv2.percona.com/v2","kind":"PerconaPGCluster","metadata":{"annotations":{},"labels":{"e2e":"major-upgrade-13-to-14"},"name":"major-upgrade-13-to-14","namespace":"kuttl-test-patient-primate"},"spec":{"backups":{"pgbackrest":{"image":"perconalab/percona-postgresql-operator:main-pgbackrest13","manual":{"options":["--type=full"],"repoName":"repo1"},"repoHost":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/data":"pgbackrest"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}}},"repos":[{"name":"repo1","schedules":{"full":"0 0 * * 6"},"volume":{"volumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}}}}]}},"crVersion":"2.9.0","extensions":{"custom":[{"name":"pg_cron","version":"1.6.6"}],"image":"docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0","imagePullPolicy":"Always","storage":{"bucket":"pg-extensions","region":"eu-central-1","secret":{"name":"aws-s3-secret"},"type":"s3"}},"image":"perconalab/percona-postgresql-operator:main-ppg13-postgres","imagePullPolicy":"Always","initContainer":{"image":"docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0"},"instances":[{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/data":"postgres"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"3Gi"}}},"name":"instance1","replicas":3}],"patroni":{"dynamicConfiguration":{"postgresql":{"parameters":{"shared_preload_libraries":"pg_cron"}}}},"pmm":{"customClusterName":"major-upgrade-13-to-14-pmm-custom-name","enabled":false,"image":"docker.io/perconalab/pmm-client:dev-latest","postgresParams":"--environment=dev-postgres","secret":"major-upgrade-13-to-14-pmm-secret","serverHost":"monitoring-service"},"postgresVersion":13,"proxy":{"pgBouncer":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/role":"pgbouncer"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}},"image":"perconalab/percona-postgresql-operator:main-pgbouncer13","replicas":3}},"users":[{"name":"postgres","password":{"type":"AlphaNumeric"}},{"name":"major-upgrade-13-to-14","password":{"type":"AlphaNumeric"}}]}} + pgv2.percona.com/patroni-version: 4.1.0 + finalizers: + - internal.percona.com/stop-watchers + labels: + e2e: major-upgrade-13-to-14 + managedFields: '[... elided field over 10 lines long ...]' name: major-upgrade-13-to-14 namespace: kuttl-test-patient-primate +spec: '[... elided field over 10 lines long ...]' status: + conditions: '[... elided field over 10 lines long ...]' + host: major-upgrade-13-to-14-pgbouncer.kuttl-test-patient-primate.svc + installedCustomExtensions: + - pg_cron + observedGeneration: 8 + patroni: + status: + systemIdentifier: "7609029758364954718" + version: 4.1.0 + patroniVersion: 4.1.0 + pgbackrest: '[... elided field over 10 lines long ...]' pgbouncer: ready: 3 size: 3 postgres: + imageID: docker.io/perconalab/percona-postgresql-operator@sha256:7b58189dfe546105ebcbb044c040e2e35bc8f356ae387b643c838ff652ee5e23 instances: - name: instance1 - ready: 3 + ready: 1 size: 3 - ready: 3 + ready: 1 size: 3 - state: ready + version: 14 + state: initializing case.go:330: resource PerconaPGCluster:kuttl-test-patient-primate/major-upgrade-13-to-14: .status.postgres.instances.ready: value mismatch, expected: 3 != actual: 1 case.go:330: command "kubectl -n ${NAMESPACE} get pod\\n for pod in $(kubectl -n ${NAMESPA..." exceeded 24 sec timeout, context deadline exceeded logger.go:42: 19:49:37 | major-upgrade-13-to-14 | major-upgrade-13-to-14 events from ns kuttl-test-patient-primate: logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:32 +0000 UTC Normal Pod pg-client-65d98588cc-cc9lq Binding Scheduled Successfully assigned kuttl-test-patient-primate/pg-client-65d98588cc-cc9lq to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:32 +0000 UTC Normal ReplicaSet.apps pg-client-65d98588cc SuccessfulCreate Created pod: pg-client-65d98588cc-cc9lq replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:32 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-65d98588cc from 0 to 1 deployment-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:33 +0000 UTC Normal Pod pg-client-65d98588cc-cc9lq.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:33 +0000 UTC Normal Pod pg-client-65d98588cc-cc9lq.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 242ms (242ms including waiting). Image size: 564266549 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:33 +0000 UTC Normal Pod pg-client-65d98588cc-cc9lq.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:33 +0000 UTC Normal Pod pg-client-65d98588cc-cc9lq.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:38 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-13-to-14 EndOfLifePostgresVersion The last minor version of Postgres 13 has been released. PG 13 will no longer receive updates. We recommend upgrading. See https://www.postgresql.org/support/versioning postgrescluster-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-6vhh-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-6vhh-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-6vhh-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-6vhh-pgdata" pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-6vhh SuccessfulCreate create Pod major-upgrade-13-to-14-instance1-6vhh-0 in StatefulSet major-upgrade-13-to-14-instance1-6vhh successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-dr44-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-dr44-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-dr44-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-dr44-pgdata" pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-dr44 SuccessfulCreate create Pod major-upgrade-13-to-14-instance1-dr44-0 in StatefulSet major-upgrade-13-to-14-instance1-dr44 successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-zwr5-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-zwr5-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-zwr5-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-zwr5-pgdata" pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:39 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-zwr5 SuccessfulCreate create Pod major-upgrade-13-to-14-instance1-zwr5-0 in StatefulSet major-upgrade-13-to-14-instance1-zwr5 successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:40 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-13-to-14-pgbouncer NoPods No matching pods found controllermanager logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:40 +0000 UTC Normal Deployment.apps major-upgrade-13-to-14-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-13-to-14-pgbouncer-675fdf8c4d from 0 to 3 deployment-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:40 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-repo-host SuccessfulCreate create Pod major-upgrade-13-to-14-repo-host-0 in StatefulSet major-upgrade-13-to-14-repo-host successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:40 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-repo1 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:40 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:40 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-patient-primate/major-upgrade-13-to-14-repo1" pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:40 +0000 UTC Normal PostgresCluster.postgres-operator.crunchydata.com major-upgrade-13-to-14 RepoHostCreated created pgBackRest repository host StatefulSet/major-upgrade-13-to-14-repo-host postgrescluster-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:41 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:41 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-jxmj default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:41 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:41 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-675fdf8c4d SuccessfulCreate Created pod: major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:41 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-675fdf8c4d SuccessfulCreate Created pod: major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:41 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-675fdf8c4d SuccessfulCreate Created pod: major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7 replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer13" in 244ms (244ms including waiting). Image size: 87482602 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer13" in 221ms (221ms including waiting). Image size: 87482602 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:42 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-6vhh-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-jxmj default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-6vhh-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-66ee6469-39ee-46a1-89fb-54386e89fbe9 pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-dr44-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-dr44-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-aa3bc67e-25f0-4930-8cfa-5755c46ca868 pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-zwr5-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-instance1-zwr5-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-635172a4-f2b6-4b3b-a7e2-2edbb9e44497 pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer13" in 243ms (243ms including waiting). Image size: 87482602 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer13" in 219ms (219ms including waiting). Image size: 87482602 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Warning Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc FailedMount MountVolume.SetUp failed for volume "pgbouncer-config" : failed to sync configmap cache: timed out waiting for the condition kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:43 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer13" in 208ms (208ms including waiting). Image size: 87482602 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer13" in 242ms (242ms including waiting). Image size: 87482602 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:44 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-c439ce78-9e23-42c0-b844-7a5acc3779a2 pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:51 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-66ee6469-39ee-46a1-89fb-54386e89fbe9" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:51 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-aa3bc67e-25f0-4930-8cfa-5755c46ca868" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:51 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-635172a4-f2b6-4b3b-a7e2-2edbb9e44497" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 225ms (225ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 244ms (244ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 222ms (222ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:52 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:53 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:53 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 254ms (254ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:53 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:53 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 260ms (260ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:53 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:53 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-13} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-13} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 244ms (244ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-13} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-13} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 253ms (253ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-13} Created Created container: extension-relocator-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 233ms (233ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:54 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-c439ce78-9e23-42c0-b844-7a5acc3779a2" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-13} Created Created container: extension-relocator-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-13} Started Started container extension-relocator-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-13} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-13} Started Started container extension-relocator-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-13} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-13} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 225ms (225ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-13} Created Created container: extension-installer-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-13} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-13} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 231ms (231ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-13} Created Created container: extension-relocator-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:55 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-13} Started Started container extension-relocator-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-13} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 255ms (255ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-13} Created Created container: extension-installer-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-13} Started Started container extension-installer-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-13} Started Started container extension-installer-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-13} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-13} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 248ms (248ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-13} Created Created container: extension-installer-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-13} Started Started container extension-installer-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest13" in 169ms (169ms including waiting). Image size: 164209021 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:56 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:57 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:57 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:57 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 241ms (241ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:57 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:57 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:57 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest13" in 226ms (227ms including waiting). Image size: 164209021 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:57 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:57 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 251ms (251ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 265ms (265ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest13" in 244ms (244ms including waiting). Image size: 164209021 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest13" in 197ms (197ms including waiting). Image size: 164209021 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:58 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 237ms (237ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 285ms (285ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg13-postgres" in 240ms (240ms including waiting). Image size: 472443954 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg13-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:19:59 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:02 +0000 UTC Warning Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:06 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-backup-qsgq-lnnrz to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:06 +0000 UTC Normal Job.batch major-upgrade-13-to-14-backup-qsgq SuccessfulCreate Created pod: major-upgrade-13-to-14-backup-qsgq-lnnrz job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:06 +0000 UTC Normal PostgresCluster.postgres-operator.crunchydata.com major-upgrade-13-to-14 StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:07 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:07 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 244ms (244ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:07 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:07 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:08 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz.spec.containers{pgbackrest} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest13" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:08 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz.spec.containers{pgbackrest} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest13" in 205ms (205ms including waiting). Image size: 164209021 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:09 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:09 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-qsgq-lnnrz.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:12 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-13-to-14 UnableToCreateStanzas command terminated with exit code 50: repo1-path = /pgbackrest/repo1 2026-02-20 19:20:12.149 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-02-20 19:20:12.159 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? postgrescluster-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:39 +0000 UTC Normal Job.batch major-upgrade-13-to-14-backup-qsgq Completed Job completed job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:20:59 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-zwr5 SuccessfulDelete delete Pod major-upgrade-13-to-14-instance1-zwr5-0 in StatefulSet major-upgrade-13-to-14-instance1-zwr5 successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:00 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-dr44 SuccessfulDelete delete Pod major-upgrade-13-to-14-instance1-dr44-0 in StatefulSet major-upgrade-13-to-14-instance1-dr44 successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-675fdf8c4d SuccessfulDelete Deleted pod: major-upgrade-13-to-14-pgbouncer-675fdf8c4d-d95pr replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-675fdf8c4d SuccessfulDelete Deleted pod: major-upgrade-13-to-14-pgbouncer-675fdf8c4d-v55mc replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-675fdf8c4d SuccessfulDelete Deleted pod: major-upgrade-13-to-14-pgbouncer-675fdf8c4d-z5bv7 replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:01 +0000 UTC Normal Deployment.apps major-upgrade-13-to-14-pgbouncer ScalingReplicaSet Scaled down replica set major-upgrade-13-to-14-pgbouncer-675fdf8c4d from 3 to 0 deployment-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:04 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-6vhh SuccessfulDelete delete Pod major-upgrade-13-to-14-instance1-6vhh-0 in StatefulSet major-upgrade-13-to-14-instance1-6vhh successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:07 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-13-to-14-set-instance1 NoPods No matching pods found controllermanager logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:08 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k Binding Scheduled Successfully assigned kuttl-test-patient-primate/13-to-14-pgdata-klq9k to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:08 +0000 UTC Normal Job.batch 13-to-14-pgdata SuccessfulCreate Created pod: 13-to-14-pgdata-klq9k job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:08 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:08 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:08 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-repo-host SuccessfulDelete delete Pod major-upgrade-13-to-14-repo-host-0 in StatefulSet major-upgrade-13-to-14-repo-host successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:09 +0000 UTC Warning Pod 13-to-14-pgdata-klq9k FailedAttachVolume Multi-Attach error for volume "pvc-66ee6469-39ee-46a1-89fb-54386e89fbe9" Volume is already exclusively attached to one node and can't be attached to another attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:27 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-66ee6469-39ee-46a1-89fb-54386e89fbe9" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:28 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:29 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 251ms (251ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:29 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:29 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:30 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-relocator-13} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:30 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-relocator-13} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 310ms (310ms including waiting). Image size: 728720234 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:30 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-relocator-13} Created Created container: extension-relocator-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:30 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-relocator-13} Started Started container extension-relocator-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:31 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-installer-13} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:31 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-installer-13} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 252ms (252ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:31 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-installer-13} Created Created container: extension-installer-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:31 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-installer-13} Started Started container extension-installer-13 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:32 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-relocator-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:32 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 222ms (222ms including waiting). Image size: 728720234 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:32 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:32 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:33 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:33 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 214ms (214ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:33 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:33 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:35 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:35 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:35 +0000 UTC Normal Pod 13-to-14-pgdata-klq9k.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:46 +0000 UTC Normal Job.batch 13-to-14-pgdata Completed Job completed job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:47 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w Binding Scheduled Successfully assigned kuttl-test-patient-primate/13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:47 +0000 UTC Normal Job.batch 13-to-14-major-upgrade-13-to-14-instance1-dr44 SuccessfulCreate Created pod: 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:47 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz Binding Scheduled Successfully assigned kuttl-test-patient-primate/13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:47 +0000 UTC Normal Job.batch 13-to-14-major-upgrade-13-to-14-instance1-zwr5 SuccessfulCreate Created pod: 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:54 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-aa3bc67e-25f0-4930-8cfa-5755c46ca868" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:56 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:56 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:56 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-dr44-rdj6w.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:57 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-635172a4-f2b6-4b3b-a7e2-2edbb9e44497" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:58 +0000 UTC Normal Job.batch 13-to-14-major-upgrade-13-to-14-instance1-dr44 Completed Job completed job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:58 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:58 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:21:58 +0000 UTC Normal Pod 13-to-14-major-upgrade-13-to-14-instance1-zwr5-lk5xz.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:01 +0000 UTC Normal Job.batch 13-to-14-major-upgrade-13-to-14-instance1-zwr5 Completed Job completed job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:05 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-6vhh-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:05 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-jxmj default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:05 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-69cbfccf44 SuccessfulCreate Created pod: major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:05 +0000 UTC Normal Deployment.apps major-upgrade-13-to-14-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-13-to-14-pgbouncer-69cbfccf44 from 0 to 3 deployment-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:05 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" in 253ms (253ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" in 235ms (235ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-69cbfccf44 SuccessfulCreate Created pod: major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8 replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:06 +0000 UTC Normal ReplicaSet.apps major-upgrade-13-to-14-pgbouncer-69cbfccf44 SuccessfulCreate Created pod: major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz replicaset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" in 244ms (244ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-d6wt8.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" in 234ms (234ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:07 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-lq8qz.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:08 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" in 235ms (235ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:08 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:08 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:08 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:08 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer14" in 246ms (246ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:08 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:08 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbouncer-69cbfccf44-cw9gz.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:13 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-c439ce78-9e23-42c0-b844-7a5acc3779a2" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:14 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:14 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 204ms (204ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:14 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:14 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:15 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-66ee6469-39ee-46a1-89fb-54386e89fbe9" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:15 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:15 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 225ms (225ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:15 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:15 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 204ms (204ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 197ms (197ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:16 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:17 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 239ms (239ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:17 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:17 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:17 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:18 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 249ms (249ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:18 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:18 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:18 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:19 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 245ms (245ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:19 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:19 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:20 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:21 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 247ms (247ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:21 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:21 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:21 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:22 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 281ms (281ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:22 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:22 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:22 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:23 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 250ms (250ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:23 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:23 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:23 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:28 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-zwr5-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-jxmj default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:29 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-dr44-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:31 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-backup-6zdd-dcj2t to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:31 +0000 UTC Normal Job.batch major-upgrade-13-to-14-backup-6zdd SuccessfulCreate Created pod: major-upgrade-13-to-14-backup-6zdd-dcj2t job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:32 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:32 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 220ms (220ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:32 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:32 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:32 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-635172a4-f2b6-4b3b-a7e2-2edbb9e44497" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:34 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:34 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 216ms (216ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:34 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:34 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-6zdd-dcj2t.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 252ms (252ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-aa3bc67e-25f0-4930-8cfa-5755c46ca868" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 247ms (247ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:37 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:37 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:37 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 239ms (239ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:37 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:37 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:38 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 234ms (234ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:38 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:38 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:38 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:38 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 239ms (239ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:38 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:38 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:39 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:39 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 250ms (250ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:39 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:39 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:40 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:40 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 236ms (236ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:40 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:40 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:40 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:40 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 257ms (257ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:40 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:40 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 238ms (238ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 251ms (251ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:41 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:43 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:43 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 244ms (244ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:43 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:43 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:44 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:44 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 246ms (246ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:44 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:44 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:44 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:44 +0000 UTC Warning Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:46 +0000 UTC Normal Job.batch major-upgrade-13-to-14-backup-6zdd Completed Job completed job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:22:47 +0000 UTC Warning Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:05 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:05 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:05 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-repo2 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:06 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-repo2 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:06 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-repo2 Provisioning External provisioner is provisioning volume for claim "kuttl-test-patient-primate/major-upgrade-13-to-14-repo2" pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:10 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:10 +0000 UTC Normal PersistentVolumeClaim major-upgrade-13-to-14-repo2 ProvisioningSucceeded Successfully provisioned volume pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97 pd.csi.storage.gke.io_gke-b018d7cba7124f90ac1e-2dfd-1929-vm_ef119f0a-40b9-4d64-a746-148218c773a4 logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:18 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-df461fc6-b96f-4338-a96d-e8f61a7abd97" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:19 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:19 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 209ms (209ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:19 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:20 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:21 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:21 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 191ms (191ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:21 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:21 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:22 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:22 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 215ms (215ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:22 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:22 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:22 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:22 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 226ms (226ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:22 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:22 +0000 UTC Normal Pod major-upgrade-13-to-14-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:38 +0000 UTC Normal Job.batch major-upgrade-13-to-14-backup-gl5p SuccessfulCreate Created pod: major-upgrade-13-to-14-backup-gl5p-4zd6n job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:39 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-backup-gl5p-4zd6n to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:39 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:39 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 222ms (222ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:39 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:39 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:40 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:40 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 198ms (198ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:40 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:40 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-gl5p-4zd6n.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:49 +0000 UTC Normal Job.batch major-upgrade-13-to-14-backup-gl5p Completed Job completed job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:55 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-backup-96kr-qw2pf to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:55 +0000 UTC Normal Job.batch major-upgrade-13-to-14-backup-96kr SuccessfulCreate Created pod: major-upgrade-13-to-14-backup-96kr-qw2pf job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:56 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:56 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 253ms (253ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:56 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:56 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:57 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:57 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 206ms (207ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:57 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:23:57 +0000 UTC Normal Pod major-upgrade-13-to-14-backup-96kr-qw2pf.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:06 +0000 UTC Normal Job.batch major-upgrade-13-to-14-backup-96kr Completed Job completed job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:37 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-pgbackrest-restore-q26hz to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:37 +0000 UTC Normal Job.batch major-upgrade-13-to-14-pgbackrest-restore SuccessfulCreate Created pod: major-upgrade-13-to-14-pgbackrest-restore-q26hz job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:52 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-66ee6469-39ee-46a1-89fb-54386e89fbe9" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:54 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:54 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest14" in 212ms (212ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:54 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:54 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:54 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz.spec.containers{pgbackrest-restore} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:54 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz.spec.containers{pgbackrest-restore} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 248ms (248ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:54 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz.spec.containers{pgbackrest-restore} Created Created container: pgbackrest-restore kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:24:55 +0000 UTC Normal Pod major-upgrade-13-to-14-pgbackrest-restore-q26hz.spec.containers{pgbackrest-restore} Started Started container pgbackrest-restore kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:04 +0000 UTC Normal Job.batch major-upgrade-13-to-14-pgbackrest-restore Completed Job completed job-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:05 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-6vhh-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-lqr5 default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:05 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-6vhh SuccessfulCreate create Pod major-upgrade-13-to-14-instance1-6vhh-0 in StatefulSet major-upgrade-13-to-14-instance1-6vhh successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:08 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:08 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 219ms (219ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:08 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:08 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:09 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:10 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 242ms (242ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:10 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:10 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:10 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:11 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 220ms (220ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:11 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:11 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:12 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:13 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 220ms (220ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:13 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:13 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:13 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:14 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 219ms (219ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:14 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:14 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:14 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:15 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 228ms (228ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:15 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:15 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:15 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-6vhh-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:21 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-dr44-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-qhql default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:21 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-dr44 SuccessfulCreate create Pod major-upgrade-13-to-14-instance1-dr44-0 in StatefulSet major-upgrade-13-to-14-instance1-dr44 successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:21 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0 Binding Scheduled Successfully assigned kuttl-test-patient-primate/major-upgrade-13-to-14-instance1-zwr5-0 to gke-jen-pg-1454-bcdf84ed-default-pool-d97d7f38-jxmj default-scheduler logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:21 +0000 UTC Normal StatefulSet.apps major-upgrade-13-to-14-instance1-zwr5 SuccessfulCreate create Pod major-upgrade-13-to-14-instance1-zwr5-0 in StatefulSet major-upgrade-13-to-14-instance1-zwr5 successful statefulset-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:28 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-635172a4-f2b6-4b3b-a7e2-2edbb9e44497" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:29 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-aa3bc67e-25f0-4930-8cfa-5755c46ca868" attachdetach-controller logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:29 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:30 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:30 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 265ms (265ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:30 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:30 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:30 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 253ms (253ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:30 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:30 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:30 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:31 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:31 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 221ms (221ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:31 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:31 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:31 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 232ms (232ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:31 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:31 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:31 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:32 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:32 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 229ms (229ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:32 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:32 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:32 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 220ms (220ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:32 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:32 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:33 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:33 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 239ms (239ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:33 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:33 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:33 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:34 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:34 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 223ms (223ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:34 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:34 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 221ms (221ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:34 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:34 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:34 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 216ms (216ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 189ms (189ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:35 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" in 266ms (266ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:36 +0000 UTC Normal Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:38 +0000 UTC Warning Pod major-upgrade-13-to-14-instance1-dr44-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | 2026-02-20 19:25:40 +0000 UTC Warning Pod major-upgrade-13-to-14-instance1-zwr5-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:49:37 | major-upgrade-13-to-14 | Deleting namespace "kuttl-test-patient-primate" case.go:240: waiting for namespace "kuttl-test-patient-primate" to be deleted timed out: failed to check deletion of namespace "kuttl-test-patient-primate": context deadline exceeded === NAME kuttl harness.go:404: run tests finished harness.go:511: cleaning up harness.go:568: removing temp folder: "" --- FAIL: kuttl (2430.55s) --- FAIL: kuttl/harness (0.00s) --- FAIL: kuttl/harness/major-upgrade-13-to-14 (2429.75s) FAIL