=== RUN kuttl harness.go:460: starting setup harness.go:258: running tests using configured kubeconfig. harness.go:281: Successful connection to cluster at: https://34.122.216.80 harness.go:366: running tests harness.go:77: going to run test suite with timeout of 600 seconds for each step harness.go:378: testsuite: e2e-tests/tests has 36 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-14-to-15 === PAUSE kuttl/harness/major-upgrade-14-to-15 === CONT kuttl/harness/major-upgrade-14-to-15 logger.go:42: 19:11:36 | major-upgrade-14-to-15 | Creating namespace "kuttl-test-safe-skylark" logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | + source ../../functions logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ realpath ../../.. logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ pwd logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-14-to-15 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++ test_name=major-upgrade-14-to-15 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ command -v oc logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ PG_VER=18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ printenv logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ which gdate logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ which date logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ which gsed logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++++ which sed logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | ++ oc get projects logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | + init_temp_dir logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | + deploy_operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | + local cw_prefix= logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | + destroy_operator logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 19:11:36 | major-upgrade-14-to-15/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 19:11:37 | major-upgrade-14-to-15/0-deploy-operator | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 19:11:37 | major-upgrade-14-to-15/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 19:11:37 | major-upgrade-14-to-15/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 logger.go:42: 19:11:37 | major-upgrade-14-to-15/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 19:11:37 | major-upgrade-14-to-15/0-deploy-operator | namespace "pg-operator" force deleted logger.go:42: 19:11:48 | major-upgrade-14-to-15/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 19:11:48 | major-upgrade-14-to-15/0-deploy-operator | + create_namespace pg-operator logger.go:42: 19:11:48 | major-upgrade-14-to-15/0-deploy-operator | + local namespace=pg-operator logger.go:42: 19:11:48 | major-upgrade-14-to-15/0-deploy-operator | + [[ -n '' ]] logger.go:42: 19:11:48 | major-upgrade-14-to-15/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 19:11:49 | major-upgrade-14-to-15/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 19:11:49 | major-upgrade-14-to-15/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 19:11:50 | major-upgrade-14-to-15/0-deploy-operator | namespace/pg-operator created logger.go:42: 19:11:50 | major-upgrade-14-to-15/0-deploy-operator | + cw_prefix=cw- logger.go:42: 19:11:50 | major-upgrade-14-to-15/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/crd.yaml logger.go:42: 19:11:51 | major-upgrade-14-to-15/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:11:51 | major-upgrade-14-to-15/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 19:11:53 | major-upgrade-14-to-15/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 19:11:53 | major-upgrade-14-to-15/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 19:11:54 | major-upgrade-14-to-15/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 19:11:54 | major-upgrade-14-to-15/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:11:54 | major-upgrade-14-to-15/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:11:56 | major-upgrade-14-to-15/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.postgres-operator.crunchydata.com serverside-applied logger.go:42: 19:11:56 | major-upgrade-14-to-15/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cw-rbac.yaml logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | + local disable_telemetry=true logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | + '[' major-upgrade-14-to-15 == telemetry-transfer ']' logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cw-operator.yaml logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 19:11:57 | major-upgrade-14-to-15/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 19:11:58 | major-upgrade-14-to-15/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 19:11:58 | major-upgrade-14-to-15/0-deploy-operator | + deploy_client logger.go:42: 19:11:58 | major-upgrade-14-to-15/0-deploy-operator | + kubectl -n kuttl-test-safe-skylark apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf/client.yaml logger.go:42: 19:11:59 | major-upgrade-14-to-15/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 19:11:59 | major-upgrade-14-to-15/0-deploy-operator | + deploy_s3_secrets logger.go:42: 19:11:59 | major-upgrade-14-to-15/0-deploy-operator | + set +o xtrace logger.go:42: 19:12:01 | major-upgrade-14-to-15/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 19:12:01 | major-upgrade-14-to-15/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 19:12:01 | major-upgrade-14-to-15/0-deploy-operator | secret/azure-secret created logger.go:42: 19:12:03 | major-upgrade-14-to-15/0-deploy-operator | secret/minio-secret created logger.go:42: 19:12:03 | major-upgrade-14-to-15/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 19:12:03 | major-upgrade-14-to-15/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 19:12:04 | major-upgrade-14-to-15/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 19:12:04 | major-upgrade-14-to-15/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 19:12:04 | major-upgrade-14-to-15/0-deploy-operator | ASSERT PASS logger.go:42: 19:12:04 | major-upgrade-14-to-15/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | starting test step 1-create-cluster logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions get_cr \ | yq eval ' .spec.postgresVersion = 14 | .spec.image = "perconalab/percona-postgresql-operator:main-ppg14-postgres" | .spec.instances[0].dataVolumeClaimSpec.resources.requests.storage = "3Gi" | .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-pgbouncer14" | .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-pgbackrest14" | .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = "pg_cron" | .spec.extensions.custom += [{"name": "pg_cron", "version": "1.6.6"}]' \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + source ../../functions logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ realpath ../../.. logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ pwd logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-14-to-15 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++ test_name=major-upgrade-14-to-15 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ command -v oc logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export PG_VER=18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ PG_VER=18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ printenv logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ which gdate logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ which date logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ which gsed logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++++ which sed logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ++ oc get projects logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + get_cr logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + local cr_name= logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + '[' -z ']' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + cr_name=major-upgrade-14-to-15 logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + local repo_path= logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + local source_path= logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + yq eval ' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.postgresVersion = 14 | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.image = "perconalab/percona-postgresql-operator:main-ppg14-postgres" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.instances[0].dataVolumeClaimSpec.resources.requests.storage = "3Gi" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-pgbouncer14" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + yq eval ' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .metadata.name = "major-upgrade-14-to-15" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .metadata.labels = {"e2e":"major-upgrade-14-to-15"} | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.postgresVersion = 18 | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-pgbackrest14" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = "pg_cron" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.users += [{"name":"major-upgrade-14-to-15","password":{"type":"AlphaNumeric"}}] | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.extensions.custom += [{"name": "pg_cron", "version": "1.6.6"}]' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.pmm.image = "docker.io/perconalab/pmm-client:dev-latest" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.pmm.secret = "major-upgrade-14-to-15-pmm-secret" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.pmm.customClusterName = "major-upgrade-14-to-15-pmm-custom-name" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.pmm.postgresParams = "--environment=dev-postgres" logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ' /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy/cr.yaml logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + kubectl -n kuttl-test-safe-skylark apply -f - logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + [[ -n '' ]] logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + case $test_name in logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + yq eval -i ' logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.extensions.image = "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.extensions.imagePullPolicy = "Always" | logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | .spec.extensions.storage = {"type": "s3", "bucket": "pg-extensions", "region": "eu-central-1", "secret": {"name": "aws-s3-secret"}} logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | ' /tmp/kuttl/pg/major-upgrade-14-to-15/cr.yaml logger.go:42: 19:12:04 | major-upgrade-14-to-15/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-14-to-15/cr.yaml logger.go:42: 19:12:05 | major-upgrade-14-to-15/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 created logger.go:42: 19:13:08 | major-upgrade-14-to-15/1-create-cluster | test step completed 1-create-cluster logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | starting test step 2-write-data logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:$(get_psql_user_pass major-upgrade-14-to-15-pguser-postgres)@$(get_psql_user_host major-upgrade-14-to-15-pguser-postgres)" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:$(get_psql_user_pass major-upgrade-14-to-15-pguser-postgres)@$(get_psql_user_host major-upgrade-14-to-15-pguser-postgres)" run_psql_local \ '\c postgres \\\ CREATE EXTENSION pg_cron' \ "postgres:$(get_psql_user_pass major-upgrade-14-to-15-pguser-postgres)@$(get_psql_user_host major-upgrade-14-to-15-pguser-postgres)"] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | + source ../../functions logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ realpath ../../.. logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ pwd logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-14-to-15 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++ test_name=major-upgrade-14-to-15 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ command -v oc logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ -n '' ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export PG_VER=18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ PG_VER=18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ PGOV1_VER=14 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ printenv logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ which gdate logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ which date logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ date=/usr/sbin/date logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ which gsed logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++++ which sed logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++ oc get projects logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++ get_psql_user_pass major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:08 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | ++ get_psql_user_host major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | + local uri=postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | + local driver=postgres logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | ++ get_client_pod logger.go:42: 19:13:09 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:13:10 | major-upgrade-14-to-15/2-write-data | + kubectl -n kuttl-test-safe-skylark exec pg-client-65d98588cc-pgz5b -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc'\''' logger.go:42: 19:13:11 | major-upgrade-14-to-15/2-write-data | ++ get_psql_user_pass major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:11 | major-upgrade-14-to-15/2-write-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:11 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:13:11 | major-upgrade-14-to-15/2-write-data | ++ get_psql_user_host major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:11 | major-upgrade-14-to-15/2-write-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:11 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:13:12 | major-upgrade-14-to-15/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:13:12 | major-upgrade-14-to-15/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 19:13:12 | major-upgrade-14-to-15/2-write-data | + local uri=postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:13:12 | major-upgrade-14-to-15/2-write-data | + local driver=postgres logger.go:42: 19:13:12 | major-upgrade-14-to-15/2-write-data | ++ get_client_pod logger.go:42: 19:13:12 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:13:12 | major-upgrade-14-to-15/2-write-data | + kubectl -n kuttl-test-safe-skylark exec pg-client-65d98588cc-pgz5b -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc'\''' logger.go:42: 19:13:13 | major-upgrade-14-to-15/2-write-data | ++ get_psql_user_pass major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:13 | major-upgrade-14-to-15/2-write-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:13 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | ++ get_psql_user_host major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | + run_psql_local '\c postgres \\\ CREATE EXTENSION pg_cron' postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | + local 'command=\c postgres \\\ CREATE EXTENSION pg_cron' logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | + local uri=postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | + local driver=postgres logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | ++ get_client_pod logger.go:42: 19:13:14 | major-upgrade-14-to-15/2-write-data | ++ kubectl -n kuttl-test-safe-skylark get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:13:15 | major-upgrade-14-to-15/2-write-data | + kubectl -n kuttl-test-safe-skylark exec pg-client-65d98588cc-pgz5b -- bash -c 'printf '\''\c postgres \\\ CREATE EXTENSION pg_cron\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc'\''' logger.go:42: 19:13:16 | major-upgrade-14-to-15/2-write-data | test step completed 2-write-data logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | starting test step 3-read-from-primary logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-14-to-15-pguser-postgres)@$(get_psql_user_host major-upgrade-14-to-15-pguser-postgres)") kubectl create configmap -n "${NAMESPACE}" 03-read-from-primary --from-literal=data="${data}"] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | + source ../../functions logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ realpath ../../.. logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ pwd logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-14-to-15 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++ test_name=major-upgrade-14-to-15 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ command -v oc logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ ! -n '' ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ -n '' ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export PG_VER=18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ PG_VER=18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export BUCKET=pg-operator-testing logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ BUCKET=pg-operator-testing logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export PGOV1_VER=14 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ PGOV1_VER=14 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ MINIO_VER=5.4.0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ REGISTRY_NAME=docker.io logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ printenv logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ grep -E '^IMAGE' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ awk -F= '{print $1}' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ which gdate logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ which date logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ date=/usr/sbin/date logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ which gsed logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++++ which sed logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ sed=/usr/sbin/sed logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | ++ oc get projects logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ get_psql_user_pass major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:16 | major-upgrade-14-to-15/3-read-from-primary | +++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | +++ get_psql_user_host major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | +++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | +++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | ++ run_psql_local '\c myapp \\\ SELECT * from myApp;' postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | ++ local 'command=\c myapp \\\ SELECT * from myApp;' logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | ++ local uri=postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | ++ local driver=postgres logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | +++ get_client_pod logger.go:42: 19:13:17 | major-upgrade-14-to-15/3-read-from-primary | +++ kubectl -n kuttl-test-safe-skylark get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:13:18 | major-upgrade-14-to-15/3-read-from-primary | ++ kubectl -n kuttl-test-safe-skylark exec pg-client-65d98588cc-pgz5b -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc'\''' logger.go:42: 19:13:19 | major-upgrade-14-to-15/3-read-from-primary | + data=' 100500' logger.go:42: 19:13:19 | major-upgrade-14-to-15/3-read-from-primary | + kubectl create configmap -n kuttl-test-safe-skylark 03-read-from-primary '--from-literal=data= 100500' logger.go:42: 19:13:19 | major-upgrade-14-to-15/3-read-from-primary | configmap/03-read-from-primary created logger.go:42: 19:13:20 | major-upgrade-14-to-15/3-read-from-primary | test step completed 3-read-from-primary logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | starting test step 4-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c set -o errexit set -o xtrace source ../../functions kubectl apply -n ${NAMESPACE} -f - <<-EOF apiVersion: pgv2.percona.com/v2 kind: PerconaPGUpgrade metadata: name: 14-to-15 spec: postgresClusterName: major-upgrade-14-to-15 image: ${IMAGE_UPGRADE} fromPostgresVersion: 14 toPostgresVersion: 15 toPostgresImage: $(get_container_image "postgres" 15) toPgBouncerImage: $(get_container_image "pgbouncer" 15) toPgBackRestImage: $(get_container_image "pgbackrest" 15) EOF] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | + source ../../functions logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ realpath ../../.. logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ pwd logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-14-to-15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ test_name=major-upgrade-14-to-15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ command -v oc logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ ! -n '' ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ -n '' ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export PG_VER=18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ PG_VER=18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export BUCKET=pg-operator-testing logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ BUCKET=pg-operator-testing logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export PGOV1_VER=14 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ PGOV1_VER=14 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export MINIO_VER=5.4.0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ MINIO_VER=5.4.0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ REGISTRY_NAME=docker.io logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ printenv logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ grep -E '^IMAGE' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ awk -F= '{print $1}' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ which gdate logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ which date logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ date=/usr/sbin/date logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ which gsed logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++++ which sed logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | +++ sed=/usr/sbin/sed logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ oc get projects logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | + kubectl apply -n kuttl-test-safe-skylark -f - logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ get_container_image postgres 15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local component=postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local pgVersion=15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ operatorVersion=main logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ [[ postgres == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ [[ postgres == \p\g\b\a\c\k\r\e\s\t ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ get_container_image pgbouncer 15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local component=pgbouncer logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local pgVersion=15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ operatorVersion=main logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ [[ pgbouncer == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ get_container_image pgbackrest 15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local component=pgbackrest logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local pgVersion=15 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ local operatorVersion=PR-1454-bcdf84ed0 logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ [[ ! PR-1454-bcdf84ed0 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ operatorVersion=main logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ [[ pgbackrest == \p\g\b\o\u\n\c\e\r ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ [[ pgbackrest == \p\g\b\a\c\k\r\e\s\t ]] logger.go:42: 19:13:20 | major-upgrade-14-to-15/4-upgrade | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15 logger.go:42: 19:13:21 | major-upgrade-14-to-15/4-upgrade | perconapgupgrade.pgv2.percona.com/14-to-15 created logger.go:42: 19:13:22 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc ready 3 3 77s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 49s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 75s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-s7px-0 4/4 Running 0 76s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-w64d-0 4/4 Running 0 76s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd 2/2 Running 0 74s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s 2/2 Running 0 74s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj 2/2 Running 0 74s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 75s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 83s logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:13:23 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 50s logger.go:42: 19:13:30 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc stopping 3 3 85s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 57s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 83s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-s7px-0 0/4 Completed 0 84s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-w64d-0 0/4 Completed 0 84s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd 2/2 Running 0 82s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s 2/2 Running 0 82s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj 2/2 Running 0 82s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 83s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 91s logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:13:31 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 58s logger.go:42: 19:13:37 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc stopping 1 0 93s logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 65s logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 91s logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 99s logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:13:38 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 65s logger.go:42: 19:13:44 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc paused 0 0 100s logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 72s logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 0/2 Completed 0 98s logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 106s logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:13:45 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 72s logger.go:42: 19:13:51 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc paused 0 0 107s logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Init:0/5 0 5s logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 79s logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 113s logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Running 0/1 5s 5s logger.go:42: 19:13:52 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 79s logger.go:42: 19:13:58 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc paused 0 0 114s logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Init:2/5 0 12s logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 86s logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Running 0/1 12s 12s logger.go:42: 19:13:59 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 86s logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc paused 0 0 2m1s logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 1/1 Running 0 19s logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 93s logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m7s logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Running 0/1 19s 19s logger.go:42: 19:14:06 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 93s logger.go:42: 19:14:13 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc paused 0 0 2m8s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 ContainerCreating 0 0s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 ContainerCreating 0 0s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 26s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 100s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m14s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Running 0/1 1s 1s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Running 0/1 1s 1s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 27s logger.go:42: 19:14:14 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 101s logger.go:42: 19:14:20 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc paused 0 0 2m15s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 ContainerCreating 0 7s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 ContainerCreating 0 7s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 33s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 107s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m21s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Running 0/1 8s 8s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Running 0/1 8s 8s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 34s logger.go:42: 19:14:21 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-mgrp Complete 1/1 31s 108s logger.go:42: 19:14:27 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc paused 0 0 2m23s logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 15s logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 15s logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 41s logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-mgrp-d5mhv 0/1 Completed 0 115s logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m29s logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 15s logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 15s logger.go:42: 19:14:28 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 41s logger.go:42: 19:14:34 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc initializing 0 0 2m30s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 22s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 22s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 48s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 0/4 Init:0/5 0 1s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 0/2 ContainerCreating 0 0s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 0/2 ContainerCreating 0 0s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 0/2 ContainerCreating 0 0s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 0/2 Init:0/2 0 1s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m36s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 22s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 22s logger.go:42: 19:14:35 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 48s logger.go:42: 19:14:42 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc initializing 0 3 2m37s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 29s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 29s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 55s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 0/4 Init:0/5 0 8s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 7s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 7s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 7s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 0/2 Init:0/2 0 8s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m43s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 30s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 30s logger.go:42: 19:14:43 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 56s logger.go:42: 19:14:49 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc initializing 0 3 2m44s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 37s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 37s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 63s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 0/4 PodInitializing 0 16s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 15s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 15s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 15s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 0/2 PodInitializing 0 16s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m51s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 37s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 37s logger.go:42: 19:14:50 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 63s logger.go:42: 19:14:56 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc ready 1 3 2m52s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 44s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 44s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 70s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-7c6q-k2tnd 0/1 Init:0/1 0 2s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 23s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-s7px-0 0/4 Init:0/5 0 3s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-w64d-0 0/4 Init:0/5 0 3s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 22s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 22s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 22s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 23s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 2m58s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 44s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 44s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 70s logger.go:42: 19:14:57 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-7c6q Running 0/1 2s 2s logger.go:42: 19:15:04 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc initializing 1 3 2m59s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 51s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 51s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 77s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-7c6q-k2tnd 1/1 Running 0 9s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 30s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-s7px-0 0/4 Init:0/5 0 10s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-w64d-0 0/4 Init:0/5 0 10s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 29s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 29s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 29s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 30s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 3m5s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 52s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 52s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 78s logger.go:42: 19:15:05 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-7c6q Running 0/1 10s 10s logger.go:42: 19:15:11 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc initializing 1 3 3m6s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 59s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 59s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 85s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-7c6q-k2tnd 0/1 Completed 0 17s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 38s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-s7px-0 0/4 PodInitializing 0 18s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-w64d-0 0/4 Init:0/5 0 18s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 37s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 37s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 37s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 38s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 3m13s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 59s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 59s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 85s logger.go:42: 19:15:12 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-7c6q Complete 1/1 12s 17s logger.go:42: 19:15:19 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc initializing 2 3 3m14s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 66s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 66s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 92s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-7c6q-k2tnd 0/1 Completed 0 24s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 45s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-s7px-0 4/4 Running 0 25s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-w64d-0 0/4 Init:3/5 0 25s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 44s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 44s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 44s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 45s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 3m20s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 67s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 67s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 93s logger.go:42: 19:15:20 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-7c6q Complete 1/1 12s 25s logger.go:42: 19:15:26 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc initializing 2 3 3m21s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 74s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 74s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 100s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-7c6q-k2tnd 0/1 Completed 0 32s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 53s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-s7px-0 4/4 Running 0 33s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 33s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 52s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 52s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 52s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 53s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 3m28s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 74s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 74s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 100s logger.go:42: 19:15:27 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-7c6q Complete 1/1 12s 32s logger.go:42: 19:15:33 | major-upgrade-14-to-15/4-upgrade | running command: [sh -c kubectl -n ${NAMESPACE} get pg,pod,job sleep 5] logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | NAME ENDPOINT STATUS POSTGRES PGBOUNCER AGE logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc ready 3 3 3m29s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | NAME READY STATUS RESTARTS AGE logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 81s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 81s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/14-to-15-pgdata-rcn2d 0/1 Completed 0 107s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-backup-7c6q-k2tnd 0/1 Completed 0 39s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 60s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-s7px-0 4/4 Running 0 40s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-instance1-w64d-0 4/4 Running 0 40s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 59s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 59s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 59s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/major-upgrade-14-to-15-repo-host-0 2/2 Running 0 60s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | pod/pg-client-65d98588cc-pgz5b 1/1 Running 0 3m35s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | NAME STATUS COMPLETIONS DURATION AGE logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-s7px Complete 1/1 15s 81s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-major-upgrade-14-to-15-instance1-w64d Complete 1/1 12s 81s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | job.batch/14-to-15-pgdata Complete 1/1 26s 107s logger.go:42: 19:15:34 | major-upgrade-14-to-15/4-upgrade | job.batch/major-upgrade-14-to-15-backup-7c6q Complete 1/1 12s 39s logger.go:42: 19:15:39 | major-upgrade-14-to-15/4-upgrade | test step completed 4-upgrade logger.go:42: 19:15:39 | major-upgrade-14-to-15/5-change-repo | starting test step 5-change-repo logger.go:42: 19:15:39 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c set -o errexit set -o xtrace kubectl patch \ -n $NAMESPACE \ perconapgcluster major-upgrade-14-to-15 \ --type='json' \ -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]'] logger.go:42: 19:15:39 | major-upgrade-14-to-15/5-change-repo | + kubectl patch -n kuttl-test-safe-skylark perconapgcluster major-upgrade-14-to-15 --type=json '-p=[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]' logger.go:42: 19:15:40 | major-upgrade-14-to-15/5-change-repo | perconapgcluster.pgv2.percona.com/major-upgrade-14-to-15 patched logger.go:42: 19:15:41 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:41 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:15:41 | major-upgrade-14-to-15/5-change-repo | name: repo1 logger.go:42: 19:15:41 | major-upgrade-14-to-15/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:15:41 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:15:41 | major-upgrade-14-to-15/5-change-repo | volume: pvc-fea12dab-de68-4dec-9e96-3531e72e0539 logger.go:42: 19:15:43 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:43 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:15:43 | major-upgrade-14-to-15/5-change-repo | name: repo1 logger.go:42: 19:15:43 | major-upgrade-14-to-15/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:15:43 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:15:43 | major-upgrade-14-to-15/5-change-repo | volume: pvc-fea12dab-de68-4dec-9e96-3531e72e0539 logger.go:42: 19:15:44 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:45 | major-upgrade-14-to-15/5-change-repo | - name: repo2 logger.go:42: 19:15:45 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:46 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:47 | major-upgrade-14-to-15/5-change-repo | - name: repo2 logger.go:42: 19:15:47 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:48 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:48 | major-upgrade-14-to-15/5-change-repo | - name: repo2 logger.go:42: 19:15:48 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:50 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:50 | major-upgrade-14-to-15/5-change-repo | - name: repo2 logger.go:42: 19:15:50 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:51 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:52 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:15:52 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:15:52 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:52 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:15:53 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:54 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:15:54 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:15:54 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:54 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:15:55 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:56 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:15:56 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:15:56 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:56 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:15:57 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:57 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:15:57 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:15:57 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:57 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:15:58 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:15:59 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:15:59 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:15:59 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:15:59 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:00 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:01 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:01 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:01 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:01 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:02 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:03 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:03 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:03 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:03 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:04 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:04 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:04 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:04 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:04 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:06 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:06 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:06 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:06 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:06 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:07 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:08 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:08 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:08 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:08 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:09 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:10 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:10 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:10 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:10 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:11 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:11 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:11 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:11 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:11 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:13 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:13 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:13 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:13 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:13 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:14 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:15 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:15 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:15 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:15 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:16 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:17 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:17 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:17 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:17 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:18 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:19 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:19 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:19 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:19 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:20 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:20 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:20 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:20 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:20 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:21 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:22 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:22 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:22 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:22 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:23 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:24 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:24 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:24 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:24 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:25 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:26 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:26 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:26 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:26 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:27 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:27 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:27 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:27 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:27 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:28 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:29 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:29 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:29 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:29 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:30 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:31 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:31 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:31 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:31 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:32 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:33 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:33 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:33 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:33 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:34 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:34 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:34 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:34 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:34 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:36 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:36 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:36 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:36 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:36 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:37 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:38 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:38 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:38 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:38 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:39 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:40 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:40 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:40 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:40 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:41 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:42 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:42 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:42 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:42 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:43 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:43 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:43 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:43 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:43 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:44 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:45 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:45 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:45 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:45 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:46 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:47 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:47 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:47 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:47 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:48 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:49 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:49 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:49 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:49 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:50 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:50 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:50 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:50 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:50 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:52 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:52 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:52 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:52 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:52 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:53 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:54 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:54 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:54 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:54 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:55 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:56 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:56 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:56 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: false logger.go:42: 19:16:56 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:57 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:57 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:57 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:57 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:16:57 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:16:59 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:16:59 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:16:59 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:16:59 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:16:59 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:00 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:01 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:01 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:01 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:01 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:02 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:03 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:03 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:03 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:03 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:04 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:05 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:05 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:05 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:05 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:06 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:06 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:06 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:06 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:06 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:07 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:08 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:08 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:08 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:08 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:09 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:10 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:10 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:10 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:10 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:11 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:12 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:12 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:12 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:12 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:13 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:13 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:13 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:13 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:13 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:15 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:15 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:15 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:15 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:15 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:16 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:17 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:17 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:17 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:17 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:18 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:19 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:19 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:19 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:19 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:20 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:21 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:21 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:21 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:21 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:22 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:22 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:22 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:22 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:22 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:23 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:24 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:24 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:24 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:24 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:25 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:26 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:26 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:26 | major-upgrade-14-to-15/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:17:26 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:26 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:27 | major-upgrade-14-to-15/5-change-repo | running command: [sh -c kubectl get postgrescluster major-upgrade-14-to-15 \ -n ${NAMESPACE} \ -o yaml \ | yq eval '.status.pgbackrest.repos' -] logger.go:42: 19:17:28 | major-upgrade-14-to-15/5-change-repo | - bound: true logger.go:42: 19:17:28 | major-upgrade-14-to-15/5-change-repo | name: repo2 logger.go:42: 19:17:28 | major-upgrade-14-to-15/5-change-repo | replicaCreateBackupComplete: true logger.go:42: 19:17:28 | major-upgrade-14-to-15/5-change-repo | stanzaCreated: true logger.go:42: 19:17:28 | major-upgrade-14-to-15/5-change-repo | volume: pvc-a5e3511e-f6b3-4369-8788-89a65c13583b logger.go:42: 19:17:28 | major-upgrade-14-to-15/5-change-repo | test step completed 5-change-repo logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | starting test step 6-write-data logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' \ "postgres:$(get_psql_user_pass major-upgrade-14-to-15-pguser-postgres)@$(get_psql_user_host major-upgrade-14-to-15-pguser-postgres)"] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | + source ../../functions logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ realpath ../../.. logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ pwd logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-14-to-15 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ test_name=major-upgrade-14-to-15 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ command -v oc logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ ! -n '' ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ -n '' ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export PG_VER=18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ PG_VER=18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export PGOV1_VER=14 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ PGOV1_VER=14 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ printenv logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ grep -E '^IMAGE' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ awk -F= '{print $1}' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ which gdate logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ which date logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ date=/usr/sbin/date logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ which gsed logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++++ which sed logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | +++ sed=/usr/sbin/sed logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ oc get projects logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ get_psql_user_pass major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ get_psql_user_host major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:28 | major-upgrade-14-to-15/6-write-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:17:29 | major-upgrade-14-to-15/6-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:17:29 | major-upgrade-14-to-15/6-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' logger.go:42: 19:17:29 | major-upgrade-14-to-15/6-write-data | + local uri=postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:17:29 | major-upgrade-14-to-15/6-write-data | + local driver=postgres logger.go:42: 19:17:29 | major-upgrade-14-to-15/6-write-data | ++ get_client_pod logger.go:42: 19:17:29 | major-upgrade-14-to-15/6-write-data | ++ kubectl -n kuttl-test-safe-skylark get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:17:29 | major-upgrade-14-to-15/6-write-data | + kubectl -n kuttl-test-safe-skylark exec pg-client-65d98588cc-pgz5b -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc'\''' logger.go:42: 19:17:30 | major-upgrade-14-to-15/6-write-data | test step completed 6-write-data logger.go:42: 19:17:30 | major-upgrade-14-to-15/7-run-backup | starting test step 7-run-backup logger.go:42: 19:17:31 | major-upgrade-14-to-15/7-run-backup | PerconaPGBackup:kuttl-test-safe-skylark/backup-after-14-to-15 created logger.go:42: 19:17:54 | major-upgrade-14-to-15/7-run-backup | test step completed 7-run-backup logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | starting test step 8-read-from-primary logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-14-to-15-pguser-postgres)@$(get_psql_user_host major-upgrade-14-to-15-pguser-postgres)") kubectl create configmap -n "${NAMESPACE}" 08-read-from-primary --from-literal=data="${data}"] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | + source ../../functions logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ realpath ../../.. logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ pwd logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-14-to-15 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++ test_name=major-upgrade-14-to-15 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ command -v oc logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ ! -n '' ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ -n '' ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export PG_VER=18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ PG_VER=18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export BUCKET=pg-operator-testing logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ BUCKET=pg-operator-testing logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export PGOV1_VER=14 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ PGOV1_VER=14 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ MINIO_VER=5.4.0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ REGISTRY_NAME=docker.io logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ printenv logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ grep -E '^IMAGE' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ awk -F= '{print $1}' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ which gdate logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ which date logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ date=/usr/sbin/date logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ which gsed logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++++ which sed logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ sed=/usr/sbin/sed logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | ++ oc get projects logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ get_psql_user_pass major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ get_psql_user_host major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:54 | major-upgrade-14-to-15/8-read-from-primary | +++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:17:55 | major-upgrade-14-to-15/8-read-from-primary | ++ run_psql_local '\c myapp \\\ SELECT * from myApp;' postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:17:55 | major-upgrade-14-to-15/8-read-from-primary | ++ local 'command=\c myapp \\\ SELECT * from myApp;' logger.go:42: 19:17:55 | major-upgrade-14-to-15/8-read-from-primary | ++ local uri=postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:17:55 | major-upgrade-14-to-15/8-read-from-primary | ++ local driver=postgres logger.go:42: 19:17:55 | major-upgrade-14-to-15/8-read-from-primary | +++ get_client_pod logger.go:42: 19:17:55 | major-upgrade-14-to-15/8-read-from-primary | +++ kubectl -n kuttl-test-safe-skylark get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:17:55 | major-upgrade-14-to-15/8-read-from-primary | ++ kubectl -n kuttl-test-safe-skylark exec pg-client-65d98588cc-pgz5b -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc'\''' logger.go:42: 19:17:56 | major-upgrade-14-to-15/8-read-from-primary | + data=' 100500 logger.go:42: 19:17:56 | major-upgrade-14-to-15/8-read-from-primary | 100501' logger.go:42: 19:17:56 | major-upgrade-14-to-15/8-read-from-primary | + kubectl create configmap -n kuttl-test-safe-skylark 08-read-from-primary '--from-literal=data= 100500 logger.go:42: 19:17:56 | major-upgrade-14-to-15/8-read-from-primary | 100501' logger.go:42: 19:17:57 | major-upgrade-14-to-15/8-read-from-primary | configmap/08-read-from-primary created logger.go:42: 19:17:57 | major-upgrade-14-to-15/8-read-from-primary | test step completed 8-read-from-primary logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | starting test step 9-delete-data logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_psql_local \ '\c myapp \\\ TRUNCATE TABLE myApp' \ "postgres:$(get_psql_user_pass major-upgrade-14-to-15-pguser-postgres)@$(get_psql_user_host major-upgrade-14-to-15-pguser-postgres)"] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | + source ../../functions logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ realpath ../../.. logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++ CERT_MANAGER_VER=1.19.1 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ pwd logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/tests/major-upgrade-14-to-15 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++ test_name=major-upgrade-14-to-15 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/vars.sh logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/deploy logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/e2e-tests/conf logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-14-to-15 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export GIT_BRANCH=PR-1454 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ GIT_BRANCH=PR-1454 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ VERSION=PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ command -v oc logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ ! -n '' ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ -n '' ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export PG_VER=18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ PG_VER=18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export BUCKET=pg-operator-testing logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ BUCKET=pg-operator-testing logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export PGOV1_VER=14 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ PGOV1_VER=14 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export MINIO_VER=5.4.0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ MINIO_VER=5.4.0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ REGISTRY_NAME=docker.io logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ printenv logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ grep -E '^IMAGE' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ awk -F= '{print $1}' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_PMM3_CLIENT' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-client:3.4 == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_PMM3_SERVER' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-server:3.4 == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3.4 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-server:dev-latest == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++++ echo perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ var_value=perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == docker.io/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == percona/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ [[ perconalab/pmm-client:dev-latest == perconalab/* ]] logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ new_value=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:dev-latest logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0 logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ which gdate logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ which date logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ date=/usr/sbin/date logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ which gsed logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1454/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++++ which sed logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | +++ sed=/usr/sbin/sed logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++ oc get projects logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++ get_psql_user_pass major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:57 | major-upgrade-14-to-15/9-delete-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | ++ get_psql_user_host major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | ++ local secret_name=major-upgrade-14-to-15-pguser-postgres logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | ++ kubectl -n kuttl-test-safe-skylark get secret/major-upgrade-14-to-15-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | + run_psql_local '\c myapp \\\ TRUNCATE TABLE myApp' postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | + local 'command=\c myapp \\\ TRUNCATE TABLE myApp' logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | + local uri=postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | + local driver=postgres logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | ++ get_client_pod logger.go:42: 19:17:58 | major-upgrade-14-to-15/9-delete-data | ++ kubectl -n kuttl-test-safe-skylark get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 19:17:59 | major-upgrade-14-to-15/9-delete-data | + kubectl -n kuttl-test-safe-skylark exec pg-client-65d98588cc-pgz5b -- bash -c 'printf '\''\c myapp \\\ TRUNCATE TABLE myApp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:IFCy5qNC8j9uAPI5qMv8HkxE@major-upgrade-14-to-15-primary.kuttl-test-safe-skylark.svc'\''' logger.go:42: 19:18:00 | major-upgrade-14-to-15/9-delete-data | test step completed 9-delete-data logger.go:42: 19:18:00 | major-upgrade-14-to-15/10-run-restore | starting test step 10-run-restore logger.go:42: 19:18:00 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-14-to-15 primary name) latest_full_repo2_backup=$(kubectl -n ${NAMESPACE} exec ${primary} -- pgbackrest info --output json --log-level-console=info | jq -r '[.[] | .backup[] | select(.type == "full") | select(.database["repo-key"] == 2)][-1].label') cat </dev/null done sleep 30] logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 3m51s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 3m51s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 4m17s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 68s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 32s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 3m30s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 4/4 Running 0 3m10s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 4/4 Running 0 3m10s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 3m29s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 3m29s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 3m29s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 2m20s logger.go:42: 19:18:05 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 6m5s logger.go:42: 19:18:06 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:18:09 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 3m57s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 3m57s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 4m23s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 74s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 38s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 3m35s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 3m35s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 3m35s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 2m26s logger.go:42: 19:18:10 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 6m11s logger.go:42: 19:18:11 | major-upgrade-14-to-15/10-run-restore | No resources found in kuttl-test-safe-skylark namespace. logger.go:42: 19:18:42 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 4m29s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 4m29s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 4m55s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 106s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 70s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 0/4 Init:0/5 0 4s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbackrest-restore-r6hpr 0/1 Completed 0 32s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 4m7s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 4m7s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 4m7s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 2m58s logger.go:42: 19:18:43 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 6m43s logger.go:42: 19:18:44 | major-upgrade-14-to-15/10-run-restore | Waiting for major-upgrade-14-to-15-instance1-rvwt-0 to start running logger.go:42: 19:19:15 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 5m3s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 5m3s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 5m29s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 2m20s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 104s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 38s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 0/4 Init:0/5 0 5s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 0/4 Init:0/5 0 5s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 4m41s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 4m41s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 4m41s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 3m32s logger.go:42: 19:19:16 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 7m17s logger.go:42: 19:19:18 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:19:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:19:20 | major-upgrade-14-to-15/10-run-restore | Waiting for major-upgrade-14-to-15-instance1-s7px-0 to start running logger.go:42: 19:19:21 | major-upgrade-14-to-15/10-run-restore | Waiting for major-upgrade-14-to-15-instance1-w64d-0 to start running logger.go:42: 19:19:52 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 5m39s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 5m39s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 6m5s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 2m56s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 2m20s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 74s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 41s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 41s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 5m17s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 5m17s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 5m17s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 4m8s logger.go:42: 19:19:53 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 7m53s logger.go:42: 19:19:54 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:19:56 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:18:06.313 UTC [129] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:18:06.349 UTC [129] LOG: database system is shut down logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.361 UTC [141] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.362 UTC [141] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.362 UTC [141] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.386 UTC [141] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.397 UTC [146] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.422 UTC [146] LOG: restored log file "00000003.history" from archive logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.445 UTC [146] LOG: entering standby mode logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.469 UTC [146] LOG: restored log file "00000003.history" from archive logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.791 UTC [146] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.821 UTC [146] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.823 UTC [141] LOG: startup process (PID 146) exited with exit code 1 logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.823 UTC [141] LOG: aborting startup due to startup process failure logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.823 UTC [141] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.830 UTC [141] LOG: database system is shut down logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.749 UTC [185] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.749 UTC [185] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.750 UTC [185] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.774 UTC [185] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.781 UTC [190] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.805 UTC [190] LOG: restored log file "00000003.history" from archive logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.828 UTC [190] LOG: entering standby mode logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.851 UTC [190] LOG: restored log file "00000003.history" from archive logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.172 UTC [190] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.205 UTC [190] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.206 UTC [185] LOG: startup process (PID 190) exited with exit code 1 logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.206 UTC [185] LOG: aborting startup due to startup process failure logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.207 UTC [185] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.214 UTC [185] LOG: database system is shut down logger.go:42: 19:19:58 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.203 UTC [145] LOG: entering standby mode logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.230 UTC [145] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.554 UTC [145] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.585 UTC [145] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.587 UTC [140] LOG: startup process (PID 145) exited with exit code 1 logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.587 UTC [140] LOG: aborting startup due to startup process failure logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.587 UTC [140] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:28.596 UTC [140] LOG: database system is shut down logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.837 UTC [193] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.838 UTC [193] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.838 UTC [193] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.864 UTC [193] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.871 UTC [198] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.900 UTC [198] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.927 UTC [198] LOG: entering standby mode logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:39.954 UTC [198] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.276 UTC [198] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.314 UTC [198] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.316 UTC [193] LOG: startup process (PID 198) exited with exit code 1 logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.316 UTC [193] LOG: aborting startup due to startup process failure logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.316 UTC [193] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:40.329 UTC [193] LOG: database system is shut down logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:59.789 UTC [263] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:59.790 UTC [263] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:59.790 UTC [263] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:59.816 UTC [263] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:59.823 UTC [268] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:59.852 UTC [268] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:59.879 UTC [268] LOG: entering standby mode logger.go:42: 19:20:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:59.905 UTC [268] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:31 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 6m18s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 6m18s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 6m44s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 3m35s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 2m59s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 113s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 80s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 80s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 5m56s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 5m56s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 5m56s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 4m47s logger.go:42: 19:20:32 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 8m32s logger.go:42: 19:20:33 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:20:35 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:00.212 UTC [262] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:00.220 UTC [262] LOG: database system is shut down logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.759 UTC [306] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.760 UTC [306] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.760 UTC [306] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.785 UTC [306] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.791 UTC [311] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.815 UTC [311] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.838 UTC [311] LOG: entering standby mode logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.862 UTC [311] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.981 UTC [311] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.019 UTC [311] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.021 UTC [306] LOG: startup process (PID 311) exited with exit code 1 logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.021 UTC [306] LOG: aborting startup due to startup process failure logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.021 UTC [306] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.032 UTC [306] LOG: database system is shut down logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.306 UTC [380] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.306 UTC [380] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.306 UTC [380] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.330 UTC [380] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.337 UTC [397] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.362 UTC [397] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.386 UTC [397] LOG: entering standby mode logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.410 UTC [397] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.728 UTC [397] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.761 UTC [397] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.762 UTC [380] LOG: startup process (PID 397) exited with exit code 1 logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.762 UTC [380] LOG: aborting startup due to startup process failure logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.762 UTC [380] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.770 UTC [380] LOG: database system is shut down logger.go:42: 19:20:37 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:00.262 UTC [263] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:00.272 UTC [263] LOG: database system is shut down logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.800 UTC [314] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.801 UTC [314] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.801 UTC [314] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.826 UTC [314] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.833 UTC [319] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.864 UTC [319] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.892 UTC [319] LOG: entering standby mode logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:09.919 UTC [319] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.042 UTC [319] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.078 UTC [319] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.080 UTC [314] LOG: startup process (PID 319) exited with exit code 1 logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.080 UTC [314] LOG: aborting startup due to startup process failure logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.080 UTC [314] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:10.089 UTC [314] LOG: database system is shut down logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.397 UTC [400] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.397 UTC [400] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.397 UTC [400] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.423 UTC [400] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.431 UTC [421] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.463 UTC [421] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.491 UTC [421] LOG: entering standby mode logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.520 UTC [421] LOG: restored log file "00000003.history" from archive logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.843 UTC [421] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.877 UTC [421] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.879 UTC [400] LOG: startup process (PID 421) exited with exit code 1 logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.879 UTC [400] LOG: aborting startup due to startup process failure logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.879 UTC [400] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:20:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:30.887 UTC [400] LOG: database system is shut down logger.go:42: 19:21:10 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 6m57s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 6m57s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 7m23s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 4m14s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 3m38s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 2m32s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 119s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 119s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 6m35s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 6m35s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 6m35s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 5m26s logger.go:42: 19:21:11 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 9m11s logger.go:42: 19:21:12 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:21:14 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:50.225 UTC [469] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:50.233 UTC [469] LOG: database system is shut down logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.762 UTC [514] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.762 UTC [514] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.762 UTC [514] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.787 UTC [514] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.793 UTC [519] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.818 UTC [519] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.841 UTC [519] LOG: entering standby mode logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.865 UTC [519] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.185 UTC [519] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.218 UTC [519] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.219 UTC [514] LOG: startup process (PID 519) exited with exit code 1 logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.219 UTC [514] LOG: aborting startup due to startup process failure logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.219 UTC [514] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.227 UTC [514] LOG: database system is shut down logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.910 UTC [558] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.911 UTC [558] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.911 UTC [558] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.935 UTC [558] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.942 UTC [563] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.966 UTC [563] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.991 UTC [563] LOG: entering standby mode logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.016 UTC [563] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.335 UTC [563] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.368 UTC [563] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.370 UTC [558] LOG: startup process (PID 563) exited with exit code 1 logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.370 UTC [558] LOG: aborting startup due to startup process failure logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.370 UTC [558] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.378 UTC [558] LOG: database system is shut down logger.go:42: 19:21:16 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:50.282 UTC [477] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:50.296 UTC [477] LOG: database system is shut down logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.839 UTC [520] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.840 UTC [520] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.840 UTC [520] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.864 UTC [520] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.872 UTC [525] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.899 UTC [525] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.926 UTC [525] LOG: entering standby mode logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:20:59.954 UTC [525] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.276 UTC [525] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.313 UTC [525] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.315 UTC [520] LOG: startup process (PID 525) exited with exit code 1 logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.315 UTC [520] LOG: aborting startup due to startup process failure logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.315 UTC [520] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:00.326 UTC [520] LOG: database system is shut down logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.984 UTC [563] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.985 UTC [563] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:09.985 UTC [563] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.010 UTC [563] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.017 UTC [568] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.047 UTC [568] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.076 UTC [568] LOG: entering standby mode logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.105 UTC [568] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.428 UTC [568] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.464 UTC [568] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.465 UTC [563] LOG: startup process (PID 568) exited with exit code 1 logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.465 UTC [563] LOG: aborting startup due to startup process failure logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.465 UTC [563] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.474 UTC [563] LOG: database system is shut down logger.go:42: 19:21:49 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 7m36s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 7m36s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 8m2s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 4m53s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 4m17s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 3m11s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 2m38s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 2m38s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 7m14s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 7m14s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 7m14s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 6m5s logger.go:42: 19:21:50 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 9m50s logger.go:42: 19:21:51 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:21:53 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.370 UTC [558] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.378 UTC [558] LOG: database system is shut down logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.817 UTC [635] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.818 UTC [635] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.818 UTC [635] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.842 UTC [635] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.848 UTC [640] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.873 UTC [640] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.897 UTC [640] LOG: entering standby mode logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.921 UTC [640] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.241 UTC [640] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.273 UTC [640] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.275 UTC [635] LOG: startup process (PID 640) exited with exit code 1 logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.275 UTC [635] LOG: aborting startup due to startup process failure logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.275 UTC [635] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.282 UTC [635] LOG: database system is shut down logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:39.947 UTC [679] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:39.948 UTC [679] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:39.948 UTC [679] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:39.973 UTC [679] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:39.979 UTC [684] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.004 UTC [684] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.028 UTC [684] LOG: entering standby mode logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.052 UTC [684] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.372 UTC [684] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.405 UTC [684] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.407 UTC [679] LOG: startup process (PID 684) exited with exit code 1 logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.407 UTC [679] LOG: aborting startup due to startup process failure logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.407 UTC [679] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.414 UTC [679] LOG: database system is shut down logger.go:42: 19:21:55 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.465 UTC [563] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:10.474 UTC [563] LOG: database system is shut down logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.864 UTC [640] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.865 UTC [640] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.865 UTC [640] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.890 UTC [640] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.898 UTC [645] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.926 UTC [645] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.952 UTC [645] LOG: entering standby mode logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:29.980 UTC [645] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.302 UTC [645] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.350 UTC [645] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.353 UTC [640] LOG: startup process (PID 645) exited with exit code 1 logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.353 UTC [640] LOG: aborting startup due to startup process failure logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.353 UTC [640] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:30.371 UTC [640] LOG: database system is shut down logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:39.981 UTC [683] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:39.982 UTC [683] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:39.982 UTC [683] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.007 UTC [683] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.015 UTC [688] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.045 UTC [688] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.074 UTC [688] LOG: entering standby mode logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.103 UTC [688] LOG: restored log file "00000003.history" from archive logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.424 UTC [688] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.458 UTC [688] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.460 UTC [683] LOG: startup process (PID 688) exited with exit code 1 logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.460 UTC [683] LOG: aborting startup due to startup process failure logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.460 UTC [683] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:21:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:21:40.468 UTC [683] LOG: database system is shut down logger.go:42: 19:22:28 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 8m15s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 8m15s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 8m41s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 5m32s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 4m56s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 3m50s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 3m17s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 3m17s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 7m53s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 7m53s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 7m53s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 6m44s logger.go:42: 19:22:29 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 10m logger.go:42: 19:22:30 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:22:32 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:00.363 UTC [756] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:00.370 UTC [756] LOG: database system is shut down logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.760 UTC [825] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.760 UTC [825] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.760 UTC [825] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.785 UTC [825] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.791 UTC [830] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.816 UTC [830] LOG: restored log file "00000003.history" from archive logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.841 UTC [830] LOG: entering standby mode logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.865 UTC [830] LOG: restored log file "00000003.history" from archive logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.184 UTC [830] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.218 UTC [830] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.219 UTC [825] LOG: startup process (PID 830) exited with exit code 1 logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.219 UTC [825] LOG: aborting startup due to startup process failure logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.219 UTC [825] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.227 UTC [825] LOG: database system is shut down logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.832 UTC [869] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.833 UTC [869] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.833 UTC [869] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.858 UTC [869] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.867 UTC [874] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.896 UTC [874] LOG: restored log file "00000003.history" from archive logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.921 UTC [874] LOG: entering standby mode logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.946 UTC [874] LOG: restored log file "00000003.history" from archive logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.265 UTC [874] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.297 UTC [874] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.299 UTC [869] LOG: startup process (PID 874) exited with exit code 1 logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.299 UTC [869] LOG: aborting startup due to startup process failure logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.299 UTC [869] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.307 UTC [869] LOG: database system is shut down logger.go:42: 19:22:34 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:00.425 UTC [760] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:00.434 UTC [760] LOG: database system is shut down logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.783 UTC [827] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.784 UTC [827] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.784 UTC [827] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.809 UTC [827] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.816 UTC [832] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.844 UTC [832] LOG: restored log file "00000003.history" from archive logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.872 UTC [832] LOG: entering standby mode logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:19.901 UTC [832] LOG: restored log file "00000003.history" from archive logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.224 UTC [832] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.258 UTC [832] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.259 UTC [827] LOG: startup process (PID 832) exited with exit code 1 logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.259 UTC [827] LOG: aborting startup due to startup process failure logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.259 UTC [827] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:20.270 UTC [827] LOG: database system is shut down logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.896 UTC [871] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.897 UTC [871] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.897 UTC [871] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.922 UTC [871] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.929 UTC [876] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.960 UTC [876] LOG: restored log file "00000003.history" from archive logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:29.988 UTC [876] LOG: entering standby mode logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.014 UTC [876] LOG: restored log file "00000003.history" from archive logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.336 UTC [876] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.371 UTC [876] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.373 UTC [871] LOG: startup process (PID 876) exited with exit code 1 logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.373 UTC [871] LOG: aborting startup due to startup process failure logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.373 UTC [871] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:22:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.383 UTC [871] LOG: database system is shut down logger.go:42: 19:23:07 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 8m55s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 8m55s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 9m21s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 6m12s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 5m36s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 4m30s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 3m57s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 3m57s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 8m33s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 8m33s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 8m33s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 7m24s logger.go:42: 19:23:08 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 11m logger.go:42: 19:23:09 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:23:11 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.299 UTC [869] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.307 UTC [869] LOG: database system is shut down logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.754 UTC [946] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.755 UTC [946] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.755 UTC [946] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.780 UTC [946] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.786 UTC [951] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.810 UTC [951] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.834 UTC [951] LOG: entering standby mode logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.858 UTC [951] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.178 UTC [951] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.211 UTC [951] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.213 UTC [946] LOG: startup process (PID 951) exited with exit code 1 logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.213 UTC [946] LOG: aborting startup due to startup process failure logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.213 UTC [946] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.220 UTC [946] LOG: database system is shut down logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.758 UTC [999] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.759 UTC [999] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.759 UTC [999] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.783 UTC [999] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.789 UTC [1004] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.814 UTC [1004] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.838 UTC [1004] LOG: entering standby mode logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.863 UTC [1004] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.182 UTC [1004] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.215 UTC [1004] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.217 UTC [999] LOG: startup process (PID 1004) exited with exit code 1 logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.217 UTC [999] LOG: aborting startup due to startup process failure logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.217 UTC [999] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.225 UTC [999] LOG: database system is shut down logger.go:42: 19:23:13 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.373 UTC [871] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:30.383 UTC [871] LOG: database system is shut down logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.802 UTC [948] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.803 UTC [948] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.803 UTC [948] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.827 UTC [948] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.834 UTC [953] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.862 UTC [953] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.888 UTC [953] LOG: entering standby mode logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:49.913 UTC [953] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.234 UTC [953] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.268 UTC [953] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.270 UTC [948] LOG: startup process (PID 953) exited with exit code 1 logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.270 UTC [948] LOG: aborting startup due to startup process failure logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.270 UTC [948] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:50.284 UTC [948] LOG: database system is shut down logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.832 UTC [1000] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.832 UTC [1000] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.832 UTC [1000] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.857 UTC [1000] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.864 UTC [1005] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.892 UTC [1005] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.919 UTC [1005] LOG: entering standby mode logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:22:59.950 UTC [1005] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.275 UTC [1005] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.312 UTC [1005] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.314 UTC [1000] LOG: startup process (PID 1005) exited with exit code 1 logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.314 UTC [1000] LOG: aborting startup due to startup process failure logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.314 UTC [1000] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:00.328 UTC [1000] LOG: database system is shut down logger.go:42: 19:23:46 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 9m34s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 9m34s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 10m logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 6m51s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 6m15s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 5m9s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 4m36s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 4m36s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 9m12s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 9m12s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 9m12s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 8m3s logger.go:42: 19:23:47 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 11m logger.go:42: 19:23:48 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:23:50 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:20.218 UTC [1076] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:20.226 UTC [1076] LOG: database system is shut down logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.179 UTC [1120] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.180 UTC [1120] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.180 UTC [1120] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.204 UTC [1120] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.211 UTC [1125] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.236 UTC [1125] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.260 UTC [1125] LOG: entering standby mode logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.286 UTC [1125] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.605 UTC [1125] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.640 UTC [1125] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.644 UTC [1120] LOG: startup process (PID 1125) exited with exit code 1 logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.644 UTC [1120] LOG: aborting startup due to startup process failure logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.644 UTC [1120] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.656 UTC [1120] LOG: database system is shut down logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.752 UTC [1189] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.753 UTC [1189] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.753 UTC [1189] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.777 UTC [1189] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.784 UTC [1194] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.810 UTC [1194] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.833 UTC [1194] LOG: entering standby mode logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.858 UTC [1194] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.178 UTC [1194] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.211 UTC [1194] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.213 UTC [1189] LOG: startup process (PID 1194) exited with exit code 1 logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.213 UTC [1189] LOG: aborting startup due to startup process failure logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.213 UTC [1189] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.220 UTC [1189] LOG: database system is shut down logger.go:42: 19:23:52 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:20.350 UTC [1077] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:20.364 UTC [1077] LOG: database system is shut down logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.258 UTC [1121] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.259 UTC [1121] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.259 UTC [1121] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.284 UTC [1121] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.299 UTC [1138] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.335 UTC [1138] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.374 UTC [1138] LOG: entering standby mode logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.403 UTC [1138] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.727 UTC [1138] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.759 UTC [1138] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.761 UTC [1121] LOG: startup process (PID 1138) exited with exit code 1 logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.761 UTC [1121] LOG: aborting startup due to startup process failure logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.761 UTC [1121] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:30.770 UTC [1121] LOG: database system is shut down logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.816 UTC [1190] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.817 UTC [1190] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.817 UTC [1190] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.842 UTC [1190] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.850 UTC [1195] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.878 UTC [1195] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.904 UTC [1195] LOG: entering standby mode logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:49.930 UTC [1195] LOG: restored log file "00000003.history" from archive logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.250 UTC [1195] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.285 UTC [1195] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.286 UTC [1190] LOG: startup process (PID 1195) exited with exit code 1 logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.286 UTC [1190] LOG: aborting startup due to startup process failure logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.286 UTC [1190] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:23:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.298 UTC [1190] LOG: database system is shut down logger.go:42: 19:24:25 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 10m logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 10m logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 10m logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 7m29s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 6m53s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 5m47s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 5m14s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 5m14s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 9m50s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 9m50s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 9m50s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 8m41s logger.go:42: 19:24:26 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 12m logger.go:42: 19:24:27 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:24:29 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.213 UTC [1189] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.220 UTC [1189] LOG: database system is shut down logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.776 UTC [1241] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.777 UTC [1241] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.777 UTC [1241] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.802 UTC [1241] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.808 UTC [1246] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.833 UTC [1246] LOG: restored log file "00000003.history" from archive logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.857 UTC [1246] LOG: entering standby mode logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.881 UTC [1246] LOG: restored log file "00000003.history" from archive logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.203 UTC [1246] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.236 UTC [1246] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.238 UTC [1241] LOG: startup process (PID 1246) exited with exit code 1 logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.238 UTC [1241] LOG: aborting startup due to startup process failure logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.238 UTC [1241] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.245 UTC [1241] LOG: database system is shut down logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.789 UTC [1310] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.789 UTC [1310] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.789 UTC [1310] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.814 UTC [1310] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.820 UTC [1315] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.845 UTC [1315] LOG: restored log file "00000003.history" from archive logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.869 UTC [1315] LOG: entering standby mode logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.893 UTC [1315] LOG: restored log file "00000003.history" from archive logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.213 UTC [1315] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.246 UTC [1315] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.247 UTC [1310] LOG: startup process (PID 1315) exited with exit code 1 logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.247 UTC [1310] LOG: aborting startup due to startup process failure logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.247 UTC [1310] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:24:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.255 UTC [1310] LOG: database system is shut down logger.go:42: 19:24:32 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.286 UTC [1190] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:50.298 UTC [1190] LOG: database system is shut down logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.814 UTC [1242] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.815 UTC [1242] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.815 UTC [1242] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.840 UTC [1242] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.847 UTC [1247] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.874 UTC [1247] LOG: restored log file "00000003.history" from archive logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.900 UTC [1247] LOG: entering standby mode logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:23:59.926 UTC [1247] LOG: restored log file "00000003.history" from archive logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.246 UTC [1247] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.282 UTC [1247] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.284 UTC [1242] LOG: startup process (PID 1247) exited with exit code 1 logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.284 UTC [1242] LOG: aborting startup due to startup process failure logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.284 UTC [1242] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:00.296 UTC [1242] LOG: database system is shut down logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.842 UTC [1311] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.843 UTC [1311] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.843 UTC [1311] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.868 UTC [1311] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.876 UTC [1316] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.905 UTC [1316] LOG: restored log file "00000003.history" from archive logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.932 UTC [1316] LOG: entering standby mode logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:19.960 UTC [1316] LOG: restored log file "00000003.history" from archive logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.283 UTC [1316] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.319 UTC [1316] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.321 UTC [1311] LOG: startup process (PID 1316) exited with exit code 1 logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.321 UTC [1311] LOG: aborting startup due to startup process failure logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.321 UTC [1311] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:24:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.336 UTC [1311] LOG: database system is shut down logger.go:42: 19:25:04 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 10m logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 10m logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 11m logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 8m9s logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 7m33s logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 6m27s logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 5m54s logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 5m54s logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 10m logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 10m logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 10m logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 9m21s logger.go:42: 19:25:05 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 13m logger.go:42: 19:25:06 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:25:08 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.247 UTC [1310] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.255 UTC [1310] LOG: database system is shut down logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.751 UTC [1388] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.752 UTC [1388] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.752 UTC [1388] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.777 UTC [1388] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.783 UTC [1393] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.808 UTC [1393] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.831 UTC [1393] LOG: entering standby mode logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.856 UTC [1393] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.175 UTC [1393] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.208 UTC [1393] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.210 UTC [1388] LOG: startup process (PID 1393) exited with exit code 1 logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.210 UTC [1388] LOG: aborting startup due to startup process failure logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.210 UTC [1388] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.218 UTC [1388] LOG: database system is shut down logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.764 UTC [1457] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.764 UTC [1457] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.764 UTC [1457] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.789 UTC [1457] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.796 UTC [1462] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.822 UTC [1462] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.846 UTC [1462] LOG: entering standby mode logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.870 UTC [1462] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.191 UTC [1462] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.224 UTC [1462] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.225 UTC [1457] LOG: startup process (PID 1462) exited with exit code 1 logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.225 UTC [1457] LOG: aborting startup due to startup process failure logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.225 UTC [1457] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.234 UTC [1457] LOG: database system is shut down logger.go:42: 19:25:11 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.321 UTC [1311] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:20.336 UTC [1311] LOG: database system is shut down logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.807 UTC [1388] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.808 UTC [1388] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.808 UTC [1388] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.833 UTC [1388] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.840 UTC [1393] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.870 UTC [1393] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.898 UTC [1393] LOG: entering standby mode logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:39.925 UTC [1393] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.248 UTC [1393] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.286 UTC [1393] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.289 UTC [1388] LOG: startup process (PID 1393) exited with exit code 1 logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.289 UTC [1388] LOG: aborting startup due to startup process failure logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.289 UTC [1388] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:40.302 UTC [1388] LOG: database system is shut down logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.817 UTC [1456] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.818 UTC [1456] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.818 UTC [1456] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.843 UTC [1456] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.850 UTC [1461] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.879 UTC [1461] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.907 UTC [1461] LOG: entering standby mode logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:59.937 UTC [1461] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.260 UTC [1461] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.293 UTC [1461] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.296 UTC [1456] LOG: startup process (PID 1461) exited with exit code 1 logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.296 UTC [1456] LOG: aborting startup due to startup process failure logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.296 UTC [1456] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.310 UTC [1456] LOG: database system is shut down logger.go:42: 19:25:43 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 11m logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 11m logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 11m logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 8m48s logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 8m12s logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 7m6s logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 6m33s logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 6m33s logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 11m logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 11m logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 11m logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 10m logger.go:42: 19:25:44 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 13m logger.go:42: 19:25:46 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:25:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:25:48 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.225 UTC [1457] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:00.234 UTC [1457] LOG: database system is shut down logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.743 UTC [1534] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.743 UTC [1534] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.743 UTC [1534] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.768 UTC [1534] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.774 UTC [1539] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.799 UTC [1539] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.823 UTC [1539] LOG: entering standby mode logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.846 UTC [1539] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.965 UTC [1539] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:19.998 UTC [1539] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:20.000 UTC [1534] LOG: startup process (PID 1539) exited with exit code 1 logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:20.000 UTC [1534] LOG: aborting startup due to startup process failure logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:20.000 UTC [1534] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:20.008 UTC [1534] LOG: database system is shut down logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.783 UTC [1578] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.783 UTC [1578] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.783 UTC [1578] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.808 UTC [1578] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.814 UTC [1583] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.839 UTC [1583] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.863 UTC [1583] LOG: entering standby mode logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.887 UTC [1583] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.206 UTC [1583] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.239 UTC [1583] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.241 UTC [1578] LOG: startup process (PID 1583) exited with exit code 1 logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.241 UTC [1578] LOG: aborting startup due to startup process failure logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.241 UTC [1578] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.249 UTC [1578] LOG: database system is shut down logger.go:42: 19:25:50 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:20.307 UTC [1533] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:20.326 UTC [1533] LOG: database system is shut down logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.825 UTC [1576] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.826 UTC [1576] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.826 UTC [1576] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.851 UTC [1576] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.859 UTC [1581] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.891 UTC [1581] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.917 UTC [1581] LOG: entering standby mode logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:29.945 UTC [1581] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.267 UTC [1581] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.302 UTC [1581] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.306 UTC [1576] LOG: startup process (PID 1581) exited with exit code 1 logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.306 UTC [1576] LOG: aborting startup due to startup process failure logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.306 UTC [1576] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:30.318 UTC [1576] LOG: database system is shut down logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:49.830 UTC [1646] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:49.830 UTC [1646] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:49.830 UTC [1646] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:49.855 UTC [1646] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:49.862 UTC [1651] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:49.896 UTC [1651] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:49.926 UTC [1651] LOG: entering standby mode logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:49.958 UTC [1651] LOG: restored log file "00000003.history" from archive logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:50.280 UTC [1651] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:50.317 UTC [1651] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:50.319 UTC [1646] LOG: startup process (PID 1651) exited with exit code 1 logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:50.319 UTC [1646] LOG: aborting startup due to startup process failure logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:50.319 UTC [1646] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:25:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:50.334 UTC [1646] LOG: database system is shut down logger.go:42: 19:26:22 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 12m logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 12m logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 12m logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 9m27s logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 8m51s logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 7m45s logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 7m12s logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 7m12s logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 11m logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 11m logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 11m logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 10m logger.go:42: 19:26:23 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 14m logger.go:42: 19:26:24 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:26:26 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:50.379 UTC [1664] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:50.387 UTC [1664] LOG: database system is shut down logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.753 UTC [1708] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.754 UTC [1708] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.754 UTC [1708] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.779 UTC [1708] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.785 UTC [1713] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.810 UTC [1713] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.834 UTC [1713] LOG: entering standby mode logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.856 UTC [1713] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.175 UTC [1713] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.208 UTC [1713] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.210 UTC [1708] LOG: startup process (PID 1713) exited with exit code 1 logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.210 UTC [1708] LOG: aborting startup due to startup process failure logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.210 UTC [1708] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.218 UTC [1708] LOG: database system is shut down logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.747 UTC [1777] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.748 UTC [1777] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.748 UTC [1777] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.772 UTC [1777] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.778 UTC [1782] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.804 UTC [1782] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.828 UTC [1782] LOG: entering standby mode logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.851 UTC [1782] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.171 UTC [1782] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.203 UTC [1782] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.205 UTC [1777] LOG: startup process (PID 1782) exited with exit code 1 logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.205 UTC [1777] LOG: aborting startup due to startup process failure logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.205 UTC [1777] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.213 UTC [1777] LOG: database system is shut down logger.go:42: 19:26:29 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.895 UTC [1703] LOG: entering standby mode logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:25:59.924 UTC [1703] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.246 UTC [1703] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.281 UTC [1703] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.283 UTC [1698] LOG: startup process (PID 1703) exited with exit code 1 logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.283 UTC [1698] LOG: aborting startup due to startup process failure logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.283 UTC [1698] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:00.295 UTC [1698] LOG: database system is shut down logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.852 UTC [1766] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.853 UTC [1766] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.853 UTC [1766] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.877 UTC [1766] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.885 UTC [1771] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.915 UTC [1771] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.943 UTC [1771] LOG: entering standby mode logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:19.971 UTC [1771] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.293 UTC [1771] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.328 UTC [1771] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.331 UTC [1766] LOG: startup process (PID 1771) exited with exit code 1 logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.331 UTC [1766] LOG: aborting startup due to startup process failure logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.331 UTC [1766] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:20.345 UTC [1766] LOG: database system is shut down logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:29.884 UTC [1809] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:29.885 UTC [1809] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:29.885 UTC [1809] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:29.909 UTC [1809] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:29.917 UTC [1814] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:29.946 UTC [1814] LOG: restored log file "00000003.history" from archive logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:29.973 UTC [1814] LOG: entering standby mode logger.go:42: 19:26:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:30.001 UTC [1814] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:01 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 12m logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 12m logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 13m logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 10m logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 9m30s logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 8m24s logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 7m51s logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 7m51s logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 12m logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 12m logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 12m logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 11m logger.go:42: 19:27:02 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 15m logger.go:42: 19:27:04 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:27:05 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:27:06 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:30.228 UTC [1829] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:30.236 UTC [1829] LOG: database system is shut down logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.743 UTC [1898] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.744 UTC [1898] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.744 UTC [1898] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.768 UTC [1898] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.775 UTC [1903] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.799 UTC [1903] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.823 UTC [1903] LOG: entering standby mode logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.847 UTC [1903] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.167 UTC [1903] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.199 UTC [1903] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.201 UTC [1898] LOG: startup process (PID 1903) exited with exit code 1 logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.201 UTC [1898] LOG: aborting startup due to startup process failure logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.201 UTC [1898] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.209 UTC [1898] LOG: database system is shut down logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.747 UTC [1941] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.747 UTC [1941] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.747 UTC [1941] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.771 UTC [1941] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.778 UTC [1946] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.802 UTC [1946] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.825 UTC [1946] LOG: entering standby mode logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.851 UTC [1946] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.170 UTC [1946] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.203 UTC [1946] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.204 UTC [1941] LOG: startup process (PID 1946) exited with exit code 1 logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.204 UTC [1941] LOG: aborting startup due to startup process failure logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.204 UTC [1941] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.213 UTC [1941] LOG: database system is shut down logger.go:42: 19:27:08 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:30.363 UTC [1809] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:30.376 UTC [1809] LOG: database system is shut down logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.794 UTC [1884] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.794 UTC [1884] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.795 UTC [1884] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.820 UTC [1884] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.827 UTC [1889] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.855 UTC [1889] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.881 UTC [1889] LOG: entering standby mode logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:49.908 UTC [1889] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.229 UTC [1889] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.262 UTC [1889] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.263 UTC [1884] LOG: startup process (PID 1889) exited with exit code 1 logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.264 UTC [1884] LOG: aborting startup due to startup process failure logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.264 UTC [1884] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:50.278 UTC [1884] LOG: database system is shut down logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.786 UTC [1928] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.787 UTC [1928] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.787 UTC [1928] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.812 UTC [1928] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.819 UTC [1933] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.848 UTC [1933] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.873 UTC [1933] LOG: entering standby mode logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:26:59.901 UTC [1933] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.224 UTC [1933] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.261 UTC [1933] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.263 UTC [1928] LOG: startup process (PID 1933) exited with exit code 1 logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.263 UTC [1928] LOG: aborting startup due to startup process failure logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.263 UTC [1928] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.275 UTC [1928] LOG: database system is shut down logger.go:42: 19:27:40 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 13m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 13m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 13m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 10m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 10m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 9m3s logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 8m30s logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 8m30s logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 13m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 13m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 13m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 11m logger.go:42: 19:27:41 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 15m logger.go:42: 19:27:43 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:27:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:27:45 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.204 UTC [1941] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.213 UTC [1941] LOG: database system is shut down logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.747 UTC [2018] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.748 UTC [2018] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.748 UTC [2018] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.772 UTC [2018] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.778 UTC [2023] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.803 UTC [2023] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.826 UTC [2023] LOG: entering standby mode logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.850 UTC [2023] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.169 UTC [2023] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.201 UTC [2023] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.203 UTC [2018] LOG: startup process (PID 2023) exited with exit code 1 logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.203 UTC [2018] LOG: aborting startup due to startup process failure logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.203 UTC [2018] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.210 UTC [2018] LOG: database system is shut down logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:29.948 UTC [2062] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:29.949 UTC [2062] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:29.949 UTC [2062] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:29.973 UTC [2062] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:29.979 UTC [2067] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.004 UTC [2067] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.028 UTC [2067] LOG: entering standby mode logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.051 UTC [2067] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.371 UTC [2067] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.403 UTC [2067] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.404 UTC [2062] LOG: startup process (PID 2067) exited with exit code 1 logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.404 UTC [2062] LOG: aborting startup due to startup process failure logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.404 UTC [2062] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.412 UTC [2062] LOG: database system is shut down logger.go:42: 19:27:47 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.263 UTC [1928] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:00.275 UTC [1928] LOG: database system is shut down logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.842 UTC [2013] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.843 UTC [2013] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.843 UTC [2013] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.869 UTC [2013] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.877 UTC [2018] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.907 UTC [2018] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.935 UTC [2018] LOG: entering standby mode logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:19.962 UTC [2018] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.284 UTC [2018] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.318 UTC [2018] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.321 UTC [2013] LOG: startup process (PID 2018) exited with exit code 1 logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.321 UTC [2013] LOG: aborting startup due to startup process failure logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.321 UTC [2013] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:20.333 UTC [2013] LOG: database system is shut down logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.010 UTC [2057] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.011 UTC [2057] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.011 UTC [2057] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.036 UTC [2057] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.044 UTC [2062] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.075 UTC [2062] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.103 UTC [2062] LOG: entering standby mode logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.133 UTC [2062] LOG: restored log file "00000003.history" from archive logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.454 UTC [2062] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.490 UTC [2062] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.491 UTC [2057] LOG: startup process (PID 2062) exited with exit code 1 logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.491 UTC [2057] LOG: aborting startup due to startup process failure logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.491 UTC [2057] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:27:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:30.500 UTC [2057] LOG: database system is shut down logger.go:42: 19:28:20 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 14m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 14m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 14m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 11m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 10m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 9m42s logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 9m9s logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 9m9s logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 13m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 13m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 13m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 12m logger.go:42: 19:28:21 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 16m logger.go:42: 19:28:22 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:28:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:28:24 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:50.197 UTC [2139] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:50.205 UTC [2139] LOG: database system is shut down logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.760 UTC [2183] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.761 UTC [2183] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.761 UTC [2183] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.786 UTC [2183] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.792 UTC [2188] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.816 UTC [2188] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.841 UTC [2188] LOG: entering standby mode logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.864 UTC [2188] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.182 UTC [2188] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.215 UTC [2188] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.216 UTC [2183] LOG: startup process (PID 2188) exited with exit code 1 logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.216 UTC [2183] LOG: aborting startup due to startup process failure logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.216 UTC [2183] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.224 UTC [2183] LOG: database system is shut down logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.747 UTC [2253] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.748 UTC [2253] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.748 UTC [2253] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.772 UTC [2253] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.778 UTC [2258] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.803 UTC [2258] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.827 UTC [2258] LOG: entering standby mode logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.850 UTC [2258] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.170 UTC [2258] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.203 UTC [2258] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.204 UTC [2253] LOG: startup process (PID 2258) exited with exit code 1 logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.204 UTC [2253] LOG: aborting startup due to startup process failure logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.204 UTC [2253] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.212 UTC [2253] LOG: database system is shut down logger.go:42: 19:28:26 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:50.259 UTC [2133] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:50.270 UTC [2133] LOG: database system is shut down logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.840 UTC [2177] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.840 UTC [2177] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.840 UTC [2177] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.865 UTC [2177] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.873 UTC [2182] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.902 UTC [2182] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.928 UTC [2182] LOG: entering standby mode logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:27:59.957 UTC [2182] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.281 UTC [2182] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.317 UTC [2182] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.319 UTC [2177] LOG: startup process (PID 2182) exited with exit code 1 logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.319 UTC [2177] LOG: aborting startup due to startup process failure logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.319 UTC [2177] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:00.332 UTC [2177] LOG: database system is shut down logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.804 UTC [2255] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.805 UTC [2255] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.805 UTC [2255] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.831 UTC [2255] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.838 UTC [2260] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.870 UTC [2260] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.898 UTC [2260] LOG: entering standby mode logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:19.926 UTC [2260] LOG: restored log file "00000003.history" from archive logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.250 UTC [2260] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.285 UTC [2260] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.287 UTC [2255] LOG: startup process (PID 2260) exited with exit code 1 logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.287 UTC [2255] LOG: aborting startup due to startup process failure logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.287 UTC [2255] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:28:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.300 UTC [2255] LOG: database system is shut down logger.go:42: 19:28:59 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 14m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 14m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 15m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 12m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 11m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 10m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 9m48s logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 9m48s logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 14m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 14m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 14m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 13m logger.go:42: 19:29:00 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 17m logger.go:42: 19:29:01 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:29:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:29:03 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.275 UTC [2305] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.284 UTC [2305] LOG: database system is shut down logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.743 UTC [2373] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.743 UTC [2373] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.743 UTC [2373] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.768 UTC [2373] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.774 UTC [2378] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.799 UTC [2378] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.822 UTC [2378] LOG: entering standby mode logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.846 UTC [2378] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.164 UTC [2378] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.197 UTC [2378] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.198 UTC [2373] LOG: startup process (PID 2378) exited with exit code 1 logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.198 UTC [2373] LOG: aborting startup due to startup process failure logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.198 UTC [2373] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.206 UTC [2373] LOG: database system is shut down logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:59.758 UTC [2417] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:59.759 UTC [2417] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:59.759 UTC [2417] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:59.784 UTC [2417] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:59.790 UTC [2422] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:59.815 UTC [2422] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:59.838 UTC [2422] LOG: entering standby mode logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:59.862 UTC [2422] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:00.181 UTC [2422] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:00.215 UTC [2422] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:00.216 UTC [2417] LOG: startup process (PID 2422) exited with exit code 1 logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:00.216 UTC [2417] LOG: aborting startup due to startup process failure logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:00.216 UTC [2417] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:00.224 UTC [2417] LOG: database system is shut down logger.go:42: 19:29:05 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.287 UTC [2255] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:20.300 UTC [2255] LOG: database system is shut down logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:29.919 UTC [2307] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:29.920 UTC [2307] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:29.920 UTC [2307] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:29.945 UTC [2307] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:29.953 UTC [2312] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:29.984 UTC [2312] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.012 UTC [2312] LOG: entering standby mode logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.045 UTC [2312] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.368 UTC [2312] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.404 UTC [2312] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.406 UTC [2307] LOG: startup process (PID 2312) exited with exit code 1 logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.406 UTC [2307] LOG: aborting startup due to startup process failure logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.406 UTC [2307] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:30.415 UTC [2307] LOG: database system is shut down logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.801 UTC [2377] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.802 UTC [2377] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.802 UTC [2377] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.826 UTC [2377] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.834 UTC [2382] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.864 UTC [2382] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.892 UTC [2382] LOG: entering standby mode logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:49.921 UTC [2382] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.243 UTC [2382] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.278 UTC [2382] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.280 UTC [2377] LOG: startup process (PID 2382) exited with exit code 1 logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.280 UTC [2377] LOG: aborting startup due to startup process failure logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.280 UTC [2377] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:07 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.294 UTC [2377] LOG: database system is shut down logger.go:42: 19:29:38 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 15m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 15m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 15m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 12m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 12m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 11m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 10m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 10m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 15m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 15m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 15m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 13m logger.go:42: 19:29:39 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 17m logger.go:42: 19:29:40 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:29:42 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:00.216 UTC [2417] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:00.224 UTC [2417] LOG: database system is shut down logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:19.747 UTC [2494] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:19.748 UTC [2494] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:19.748 UTC [2494] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:19.772 UTC [2494] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:19.778 UTC [2499] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:19.803 UTC [2499] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:19.826 UTC [2499] LOG: entering standby mode logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:19.850 UTC [2499] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:20.169 UTC [2499] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:20.201 UTC [2499] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:20.202 UTC [2494] LOG: startup process (PID 2499) exited with exit code 1 logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:20.202 UTC [2494] LOG: aborting startup due to startup process failure logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:20.202 UTC [2494] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:20.211 UTC [2494] LOG: database system is shut down logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.923 UTC [2538] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.924 UTC [2538] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.924 UTC [2538] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.948 UTC [2538] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.954 UTC [2543] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.978 UTC [2543] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.002 UTC [2543] LOG: entering standby mode logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.028 UTC [2543] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.347 UTC [2543] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.381 UTC [2543] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.382 UTC [2538] LOG: startup process (PID 2543) exited with exit code 1 logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.382 UTC [2538] LOG: aborting startup due to startup process failure logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.382 UTC [2538] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.390 UTC [2538] LOG: database system is shut down logger.go:42: 19:29:44 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.280 UTC [2377] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:28:50.294 UTC [2377] LOG: database system is shut down logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:09.839 UTC [2454] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:09.839 UTC [2454] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:09.839 UTC [2454] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:09.864 UTC [2454] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:09.872 UTC [2459] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:09.902 UTC [2459] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:09.929 UTC [2459] LOG: entering standby mode logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:09.956 UTC [2459] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:10.278 UTC [2459] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:10.316 UTC [2459] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:10.319 UTC [2454] LOG: startup process (PID 2459) exited with exit code 1 logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:10.319 UTC [2454] LOG: aborting startup due to startup process failure logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:10.319 UTC [2454] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:10.333 UTC [2454] LOG: database system is shut down logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.911 UTC [2522] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.911 UTC [2522] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.911 UTC [2522] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.936 UTC [2522] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.943 UTC [2527] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.970 UTC [2527] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:29.996 UTC [2527] LOG: entering standby mode logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.021 UTC [2527] LOG: restored log file "00000003.history" from archive logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.342 UTC [2527] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.378 UTC [2527] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.380 UTC [2522] LOG: startup process (PID 2527) exited with exit code 1 logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.380 UTC [2522] LOG: aborting startup due to startup process failure logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.380 UTC [2522] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:29:46 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:30.388 UTC [2522] LOG: database system is shut down logger.go:42: 19:30:17 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 16m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 16m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 16m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 13m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 12m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 11m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 11m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 11m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 15m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 15m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 15m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 14m logger.go:42: 19:30:18 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 18m logger.go:42: 19:30:19 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:30:21 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:50.203 UTC [2624] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:50.211 UTC [2624] LOG: database system is shut down logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.896 UTC [2668] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.896 UTC [2668] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.896 UTC [2668] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.920 UTC [2668] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.927 UTC [2673] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.952 UTC [2673] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.977 UTC [2673] LOG: entering standby mode logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.002 UTC [2673] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.322 UTC [2673] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.354 UTC [2673] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.356 UTC [2668] LOG: startup process (PID 2673) exited with exit code 1 logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.356 UTC [2668] LOG: aborting startup due to startup process failure logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.356 UTC [2668] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.364 UTC [2668] LOG: database system is shut down logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.748 UTC [2737] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.748 UTC [2737] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.748 UTC [2737] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.773 UTC [2737] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.779 UTC [2742] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.804 UTC [2742] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.827 UTC [2742] LOG: entering standby mode logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.851 UTC [2742] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.171 UTC [2742] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.203 UTC [2742] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.204 UTC [2737] LOG: startup process (PID 2742) exited with exit code 1 logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.204 UTC [2737] LOG: aborting startup due to startup process failure logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.204 UTC [2737] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.212 UTC [2737] LOG: database system is shut down logger.go:42: 19:30:23 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:50.285 UTC [2598] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:50.299 UTC [2598] LOG: database system is shut down logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.996 UTC [2642] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.996 UTC [2642] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:29:59.996 UTC [2642] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.021 UTC [2642] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.028 UTC [2647] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.055 UTC [2647] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.081 UTC [2647] LOG: entering standby mode logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.109 UTC [2647] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.432 UTC [2647] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.465 UTC [2647] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.467 UTC [2642] LOG: startup process (PID 2647) exited with exit code 1 logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.467 UTC [2642] LOG: aborting startup due to startup process failure logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.467 UTC [2642] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:00.476 UTC [2642] LOG: database system is shut down logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.785 UTC [2711] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.786 UTC [2711] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.786 UTC [2711] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.810 UTC [2711] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.817 UTC [2716] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.846 UTC [2716] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.874 UTC [2716] LOG: entering standby mode logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:19.902 UTC [2716] LOG: restored log file "00000003.history" from archive logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.224 UTC [2716] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.258 UTC [2716] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.260 UTC [2711] LOG: startup process (PID 2716) exited with exit code 1 logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.260 UTC [2711] LOG: aborting startup due to startup process failure logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.260 UTC [2711] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:30:25 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.271 UTC [2711] LOG: database system is shut down logger.go:42: 19:30:56 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 16m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 16m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 17m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 14m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 13m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 12m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 11m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 11m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 16m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 16m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 16m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 15m logger.go:42: 19:30:57 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 18m logger.go:42: 19:30:58 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:31:00 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.672 UTC [2789] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.683 UTC [2789] LOG: database system is shut down logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.739 UTC [2857] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.739 UTC [2857] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.739 UTC [2857] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.763 UTC [2857] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.769 UTC [2862] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.794 UTC [2862] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.818 UTC [2862] LOG: entering standby mode logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.841 UTC [2862] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.161 UTC [2862] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.194 UTC [2862] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.195 UTC [2857] LOG: startup process (PID 2862) exited with exit code 1 logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.195 UTC [2857] LOG: aborting startup due to startup process failure logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.196 UTC [2857] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.204 UTC [2857] LOG: database system is shut down logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:59.754 UTC [2901] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:59.754 UTC [2901] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:59.754 UTC [2901] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:59.778 UTC [2901] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:59.785 UTC [2906] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:59.809 UTC [2906] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:59.832 UTC [2906] LOG: entering standby mode logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:59.856 UTC [2906] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:00.176 UTC [2906] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:00.209 UTC [2906] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:00.210 UTC [2901] LOG: startup process (PID 2906) exited with exit code 1 logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:00.210 UTC [2901] LOG: aborting startup due to startup process failure logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:00.210 UTC [2901] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:00.219 UTC [2901] LOG: database system is shut down logger.go:42: 19:31:02 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.260 UTC [2711] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:20.271 UTC [2711] LOG: database system is shut down logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.257 UTC [2763] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.258 UTC [2763] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.258 UTC [2763] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.283 UTC [2763] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.289 UTC [2769] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.325 UTC [2769] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.362 UTC [2769] LOG: entering standby mode logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.392 UTC [2769] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.714 UTC [2769] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.749 UTC [2769] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.750 UTC [2763] LOG: startup process (PID 2769) exited with exit code 1 logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.750 UTC [2763] LOG: aborting startup due to startup process failure logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.750 UTC [2763] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:30.759 UTC [2763] LOG: database system is shut down logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.793 UTC [2831] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.793 UTC [2831] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.793 UTC [2831] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.818 UTC [2831] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.827 UTC [2836] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.855 UTC [2836] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.883 UTC [2836] LOG: entering standby mode logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:49.909 UTC [2836] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.231 UTC [2836] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.265 UTC [2836] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.267 UTC [2831] LOG: startup process (PID 2836) exited with exit code 1 logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.267 UTC [2831] LOG: aborting startup due to startup process failure logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.267 UTC [2831] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:04 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.280 UTC [2831] LOG: database system is shut down logger.go:42: 19:31:35 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 17m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 17m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 17m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 14m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 14m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 12m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 12m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 12m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 17m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 17m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 17m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 15m logger.go:42: 19:31:36 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 19m logger.go:42: 19:31:37 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:31:39 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:00.210 UTC [2901] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:00.219 UTC [2901] LOG: database system is shut down logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.744 UTC [2953] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.745 UTC [2953] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.745 UTC [2953] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.770 UTC [2953] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.776 UTC [2958] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.801 UTC [2958] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.824 UTC [2958] LOG: entering standby mode logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.848 UTC [2958] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.166 UTC [2958] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.201 UTC [2958] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.202 UTC [2953] LOG: startup process (PID 2958) exited with exit code 1 logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.202 UTC [2953] LOG: aborting startup due to startup process failure logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.202 UTC [2953] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.211 UTC [2953] LOG: database system is shut down logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.770 UTC [3022] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.771 UTC [3022] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.771 UTC [3022] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.796 UTC [3022] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.802 UTC [3027] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.827 UTC [3027] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.851 UTC [3027] LOG: entering standby mode logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.875 UTC [3027] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.195 UTC [3027] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.227 UTC [3027] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.229 UTC [3022] LOG: startup process (PID 3027) exited with exit code 1 logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.229 UTC [3022] LOG: aborting startup due to startup process failure logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.229 UTC [3022] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.237 UTC [3022] LOG: database system is shut down logger.go:42: 19:31:41 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.267 UTC [2831] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:30:50.280 UTC [2831] LOG: database system is shut down logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.822 UTC [2908] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.823 UTC [2908] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.823 UTC [2908] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.847 UTC [2908] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.855 UTC [2913] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.884 UTC [2913] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.908 UTC [2913] LOG: entering standby mode logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:09.935 UTC [2913] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.255 UTC [2913] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.293 UTC [2913] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.295 UTC [2908] LOG: startup process (PID 2913) exited with exit code 1 logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.295 UTC [2908] LOG: aborting startup due to startup process failure logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.295 UTC [2908] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:10.309 UTC [2908] LOG: database system is shut down logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.873 UTC [2976] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.873 UTC [2976] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.873 UTC [2976] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.898 UTC [2976] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.905 UTC [2981] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.934 UTC [2981] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.961 UTC [2981] LOG: entering standby mode logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:29.988 UTC [2981] LOG: restored log file "00000003.history" from archive logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.310 UTC [2981] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.369 UTC [2981] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.371 UTC [2976] LOG: startup process (PID 2981) exited with exit code 1 logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.371 UTC [2976] LOG: aborting startup due to startup process failure logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.371 UTC [2976] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:31:43 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:30.385 UTC [2976] LOG: database system is shut down logger.go:42: 19:32:14 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 18m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 18m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 18m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 15m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 14m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 13m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 13m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 13m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 17m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 17m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 17m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 16m logger.go:42: 19:32:15 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 20m logger.go:42: 19:32:16 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:32:18 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:49.845 UTC [3104] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:50.164 UTC [3104] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:50.197 UTC [3104] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:50.198 UTC [3099] LOG: startup process (PID 3104) exited with exit code 1 logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:50.198 UTC [3099] LOG: aborting startup due to startup process failure logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:50.198 UTC [3099] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:50.206 UTC [3099] LOG: database system is shut down logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.755 UTC [3143] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.756 UTC [3143] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.756 UTC [3143] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.780 UTC [3143] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.786 UTC [3148] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.811 UTC [3148] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.834 UTC [3148] LOG: entering standby mode logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.858 UTC [3148] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.176 UTC [3148] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.210 UTC [3148] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.211 UTC [3143] LOG: startup process (PID 3148) exited with exit code 1 logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.211 UTC [3143] LOG: aborting startup due to startup process failure logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.211 UTC [3143] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.219 UTC [3143] LOG: database system is shut down logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.756 UTC [3221] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.756 UTC [3221] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.757 UTC [3221] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.781 UTC [3221] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.788 UTC [3226] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.813 UTC [3226] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.837 UTC [3226] LOG: entering standby mode logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.862 UTC [3226] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:20 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.181 UTC [3226] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:21 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:50.257 UTC [3052] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:50.266 UTC [3052] LOG: database system is shut down logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.823 UTC [3105] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.823 UTC [3105] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.823 UTC [3105] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.848 UTC [3105] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.856 UTC [3110] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.883 UTC [3110] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.911 UTC [3110] LOG: entering standby mode logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:31:59.938 UTC [3110] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.261 UTC [3110] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.296 UTC [3110] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.298 UTC [3105] LOG: startup process (PID 3110) exited with exit code 1 logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.298 UTC [3105] LOG: aborting startup due to startup process failure logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.298 UTC [3105] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:00.312 UTC [3105] LOG: database system is shut down logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.800 UTC [3174] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.800 UTC [3174] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.800 UTC [3174] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.825 UTC [3174] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.832 UTC [3179] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.862 UTC [3179] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.888 UTC [3179] LOG: entering standby mode logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:19.916 UTC [3179] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.238 UTC [3179] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.274 UTC [3179] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.276 UTC [3174] LOG: startup process (PID 3179) exited with exit code 1 logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.276 UTC [3174] LOG: aborting startup due to startup process failure logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.276 UTC [3174] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:22 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.290 UTC [3174] LOG: database system is shut down logger.go:42: 19:32:53 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 18m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 18m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 19m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 15m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 15m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 14m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 13m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 13m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 18m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 18m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 18m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 17m logger.go:42: 19:32:54 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 20m logger.go:42: 19:32:55 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:32:57 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.215 UTC [3221] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:20.224 UTC [3221] LOG: database system is shut down logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:29.822 UTC [3273] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:29.822 UTC [3273] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:29.822 UTC [3273] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:29.848 UTC [3273] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:29.855 UTC [3278] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:29.880 UTC [3278] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:29.905 UTC [3278] LOG: entering standby mode logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:29.930 UTC [3278] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:30.249 UTC [3278] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:30.282 UTC [3278] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:30.283 UTC [3273] LOG: startup process (PID 3278) exited with exit code 1 logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:30.283 UTC [3273] LOG: aborting startup due to startup process failure logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:30.283 UTC [3273] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:30.292 UTC [3273] LOG: database system is shut down logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.748 UTC [3342] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.748 UTC [3342] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.749 UTC [3342] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.772 UTC [3342] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.778 UTC [3347] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.803 UTC [3347] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.826 UTC [3347] LOG: entering standby mode logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.850 UTC [3347] LOG: restored log file "00000003.history" from archive logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.169 UTC [3347] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.207 UTC [3347] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.208 UTC [3342] LOG: startup process (PID 3347) exited with exit code 1 logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.208 UTC [3342] LOG: aborting startup due to startup process failure logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.208 UTC [3342] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:32:59 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.217 UTC [3342] LOG: database system is shut down logger.go:42: 19:33:00 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:30.376 UTC [3226] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:30.386 UTC [3226] LOG: database system is shut down logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.823 UTC [3296] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.824 UTC [3296] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.824 UTC [3296] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.849 UTC [3296] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.857 UTC [3301] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.886 UTC [3301] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.913 UTC [3301] LOG: entering standby mode logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:49.940 UTC [3301] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.262 UTC [3301] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.298 UTC [3301] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.300 UTC [3296] LOG: startup process (PID 3301) exited with exit code 1 logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.300 UTC [3296] LOG: aborting startup due to startup process failure logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.300 UTC [3296] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:50.314 UTC [3296] LOG: database system is shut down logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:59.791 UTC [3340] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:59.792 UTC [3340] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:59.792 UTC [3340] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:59.816 UTC [3340] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:59.823 UTC [3345] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:59.852 UTC [3345] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:59.879 UTC [3345] LOG: entering standby mode logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:32:59.906 UTC [3345] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.227 UTC [3345] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.260 UTC [3345] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.262 UTC [3340] LOG: startup process (PID 3345) exited with exit code 1 logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.262 UTC [3340] LOG: aborting startup due to startup process failure logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.262 UTC [3340] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:01 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.274 UTC [3340] LOG: database system is shut down logger.go:42: 19:33:32 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 19m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 19m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 19m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 16m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 16m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 14m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 14m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 14m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 18m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 18m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 18m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 17m logger.go:42: 19:33:33 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 21m logger.go:42: 19:33:35 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:33:36 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:33:37 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.238 UTC [3394] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.246 UTC [3394] LOG: database system is shut down logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.745 UTC [3464] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.746 UTC [3464] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.746 UTC [3464] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.770 UTC [3464] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.777 UTC [3469] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.802 UTC [3469] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.825 UTC [3469] LOG: entering standby mode logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.849 UTC [3469] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.168 UTC [3469] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.200 UTC [3469] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.202 UTC [3464] LOG: startup process (PID 3469) exited with exit code 1 logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.202 UTC [3464] LOG: aborting startup due to startup process failure logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.202 UTC [3464] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.210 UTC [3464] LOG: database system is shut down logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.833 UTC [3508] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.834 UTC [3508] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.834 UTC [3508] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.858 UTC [3508] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.864 UTC [3513] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.889 UTC [3513] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.913 UTC [3513] LOG: entering standby mode logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.936 UTC [3513] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.255 UTC [3513] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.292 UTC [3513] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.293 UTC [3508] LOG: startup process (PID 3513) exited with exit code 1 logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.293 UTC [3508] LOG: aborting startup due to startup process failure logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.293 UTC [3508] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:38 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.302 UTC [3508] LOG: database system is shut down logger.go:42: 19:33:39 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.262 UTC [3340] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:00.274 UTC [3340] LOG: database system is shut down logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.818 UTC [3417] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.819 UTC [3417] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.819 UTC [3417] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.843 UTC [3417] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.850 UTC [3422] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.878 UTC [3422] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.903 UTC [3422] LOG: entering standby mode logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:19.929 UTC [3422] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.249 UTC [3422] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.282 UTC [3422] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.284 UTC [3417] LOG: startup process (PID 3422) exited with exit code 1 logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.284 UTC [3417] LOG: aborting startup due to startup process failure logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.284 UTC [3417] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:20.297 UTC [3417] LOG: database system is shut down logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.901 UTC [3461] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.902 UTC [3461] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.902 UTC [3461] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.927 UTC [3461] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.934 UTC [3466] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.964 UTC [3466] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:29.989 UTC [3466] LOG: entering standby mode logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.017 UTC [3466] LOG: restored log file "00000003.history" from archive logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.339 UTC [3466] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.375 UTC [3466] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.377 UTC [3461] LOG: startup process (PID 3466) exited with exit code 1 logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.377 UTC [3461] LOG: aborting startup due to startup process failure logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.377 UTC [3461] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:33:40 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.386 UTC [3461] LOG: database system is shut down logger.go:42: 19:34:11 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 19m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 19m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 20m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 17m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 16m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 15m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 15m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 15m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 19m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 19m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 19m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 18m logger.go:42: 19:34:12 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 22m logger.go:42: 19:34:13 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:34:15 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:34:16 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.293 UTC [3508] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.302 UTC [3508] LOG: database system is shut down logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.751 UTC [3583] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.752 UTC [3583] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.752 UTC [3583] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.776 UTC [3583] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.783 UTC [3588] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.807 UTC [3588] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.830 UTC [3588] LOG: entering standby mode logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.854 UTC [3588] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.173 UTC [3588] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.205 UTC [3588] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.206 UTC [3583] LOG: startup process (PID 3588) exited with exit code 1 logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.206 UTC [3583] LOG: aborting startup due to startup process failure logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.206 UTC [3583] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.215 UTC [3583] LOG: database system is shut down logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.748 UTC [3627] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.749 UTC [3627] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.749 UTC [3627] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.773 UTC [3627] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.779 UTC [3632] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.804 UTC [3632] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.827 UTC [3632] LOG: entering standby mode logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.851 UTC [3632] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.169 UTC [3632] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.203 UTC [3632] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.204 UTC [3627] LOG: startup process (PID 3632) exited with exit code 1 logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.204 UTC [3627] LOG: aborting startup due to startup process failure logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.204 UTC [3627] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:17 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.212 UTC [3627] LOG: database system is shut down logger.go:42: 19:34:18 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.377 UTC [3461] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:30.386 UTC [3461] LOG: database system is shut down logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.831 UTC [3538] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.832 UTC [3538] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.832 UTC [3538] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.856 UTC [3538] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.863 UTC [3543] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.891 UTC [3543] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.916 UTC [3543] LOG: entering standby mode logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:49.940 UTC [3543] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.261 UTC [3543] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.296 UTC [3543] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.299 UTC [3538] LOG: startup process (PID 3543) exited with exit code 1 logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.299 UTC [3538] LOG: aborting startup due to startup process failure logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.299 UTC [3538] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:50.311 UTC [3538] LOG: database system is shut down logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.842 UTC [3582] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.842 UTC [3582] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.842 UTC [3582] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.867 UTC [3582] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.874 UTC [3587] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.903 UTC [3587] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.929 UTC [3587] LOG: entering standby mode logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:33:59.955 UTC [3587] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.277 UTC [3587] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.314 UTC [3587] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.319 UTC [3582] LOG: startup process (PID 3587) exited with exit code 1 logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.319 UTC [3582] LOG: aborting startup due to startup process failure logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.319 UTC [3582] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:19 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:00.333 UTC [3582] LOG: database system is shut down logger.go:42: 19:34:50 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 20m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 20m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 21m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 17m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 17m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 16m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 15m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 15m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 20m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 20m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 20m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 19m logger.go:42: 19:34:51 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 22m logger.go:42: 19:34:53 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:34:54 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:34:55 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:20.213 UTC [3703] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:20.221 UTC [3703] LOG: database system is shut down logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.828 UTC [3746] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.829 UTC [3746] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.829 UTC [3746] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.853 UTC [3746] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.859 UTC [3751] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.884 UTC [3751] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.907 UTC [3751] LOG: entering standby mode logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.930 UTC [3751] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.249 UTC [3751] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.282 UTC [3751] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.283 UTC [3746] LOG: startup process (PID 3751) exited with exit code 1 logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.283 UTC [3746] LOG: aborting startup due to startup process failure logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.283 UTC [3746] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.291 UTC [3746] LOG: database system is shut down logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.747 UTC [3815] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.748 UTC [3815] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.748 UTC [3815] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.772 UTC [3815] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.779 UTC [3820] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.803 UTC [3820] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.826 UTC [3820] LOG: entering standby mode logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.850 UTC [3820] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.168 UTC [3820] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.202 UTC [3820] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.203 UTC [3815] LOG: startup process (PID 3820) exited with exit code 1 logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.203 UTC [3815] LOG: aborting startup due to startup process failure logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.203 UTC [3815] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:56 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.211 UTC [3815] LOG: database system is shut down logger.go:42: 19:34:57 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:20.320 UTC [3659] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:20.339 UTC [3659] LOG: database system is shut down logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.866 UTC [3702] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.867 UTC [3702] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.867 UTC [3702] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.891 UTC [3702] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.898 UTC [3707] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.925 UTC [3707] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.949 UTC [3707] LOG: entering standby mode logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:29.975 UTC [3707] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.296 UTC [3707] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.331 UTC [3707] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.333 UTC [3702] LOG: startup process (PID 3707) exited with exit code 1 logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.333 UTC [3702] LOG: aborting startup due to startup process failure logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.333 UTC [3702] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:30.347 UTC [3702] LOG: database system is shut down logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.822 UTC [3771] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.823 UTC [3771] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.823 UTC [3771] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.848 UTC [3771] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.855 UTC [3776] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.885 UTC [3776] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.911 UTC [3776] LOG: entering standby mode logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:49.938 UTC [3776] LOG: restored log file "00000003.history" from archive logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.259 UTC [3776] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.295 UTC [3776] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.297 UTC [3771] LOG: startup process (PID 3776) exited with exit code 1 logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.297 UTC [3771] LOG: aborting startup due to startup process failure logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.297 UTC [3771] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:34:58 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:34:50.311 UTC [3771] LOG: database system is shut down logger.go:42: 19:35:29 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 21m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 21m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 21m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 18m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 17m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 16m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 16m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 16m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 20m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 20m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 20m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 19m logger.go:42: 19:35:30 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 23m logger.go:42: 19:35:32 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:35:33 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:35:34 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:00.200 UTC [3876] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:00.208 UTC [3876] LOG: database system is shut down logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.755 UTC [3945] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.756 UTC [3945] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.756 UTC [3945] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.780 UTC [3945] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.787 UTC [3950] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.811 UTC [3950] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.837 UTC [3950] LOG: entering standby mode logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.861 UTC [3950] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.182 UTC [3950] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.218 UTC [3950] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.220 UTC [3945] LOG: startup process (PID 3950) exited with exit code 1 logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.220 UTC [3945] LOG: aborting startup due to startup process failure logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.220 UTC [3945] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.228 UTC [3945] LOG: database system is shut down logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.791 UTC [3989] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.791 UTC [3989] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.791 UTC [3989] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.815 UTC [3989] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.822 UTC [3994] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.846 UTC [3994] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.869 UTC [3994] LOG: entering standby mode logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.894 UTC [3994] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.213 UTC [3994] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.245 UTC [3994] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.247 UTC [3989] LOG: startup process (PID 3994) exited with exit code 1 logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.247 UTC [3989] LOG: aborting startup due to startup process failure logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.247 UTC [3989] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:35 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.255 UTC [3989] LOG: database system is shut down logger.go:42: 19:35:36 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:00.293 UTC [3823] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:00.307 UTC [3823] LOG: database system is shut down logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.775 UTC [3892] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.776 UTC [3892] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.776 UTC [3892] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.800 UTC [3892] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.808 UTC [3897] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.835 UTC [3897] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.861 UTC [3897] LOG: entering standby mode logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:19.888 UTC [3897] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.210 UTC [3897] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.242 UTC [3897] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.243 UTC [3892] LOG: startup process (PID 3897) exited with exit code 1 logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.244 UTC [3892] LOG: aborting startup due to startup process failure logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.244 UTC [3892] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:20.253 UTC [3892] LOG: database system is shut down logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.865 UTC [3936] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.865 UTC [3936] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.865 UTC [3936] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.890 UTC [3936] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.898 UTC [3941] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.927 UTC [3941] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.952 UTC [3941] LOG: entering standby mode logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:29.978 UTC [3941] LOG: restored log file "00000003.history" from archive logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.299 UTC [3941] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.349 UTC [3941] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.351 UTC [3936] LOG: startup process (PID 3941) exited with exit code 1 logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.351 UTC [3936] LOG: aborting startup due to startup process failure logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.351 UTC [3936] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:35:37 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:30.362 UTC [3936] LOG: database system is shut down logger.go:42: 19:36:08 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 21m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 21m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 22m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 19m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 18m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 17m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 16m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 16m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 21m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 21m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 21m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 20m logger.go:42: 19:36:09 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 24m logger.go:42: 19:36:11 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:36:12 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:36:13 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:50.203 UTC [4066] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:50.211 UTC [4066] LOG: database system is shut down logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.755 UTC [4109] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.755 UTC [4109] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.756 UTC [4109] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.780 UTC [4109] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.787 UTC [4114] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.812 UTC [4114] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.835 UTC [4114] LOG: entering standby mode logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.858 UTC [4114] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.177 UTC [4114] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.209 UTC [4114] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.210 UTC [4109] LOG: startup process (PID 4114) exited with exit code 1 logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.210 UTC [4109] LOG: aborting startup due to startup process failure logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.210 UTC [4109] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.219 UTC [4109] LOG: database system is shut down logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.760 UTC [4153] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.760 UTC [4153] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.760 UTC [4153] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.785 UTC [4153] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.791 UTC [4158] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.816 UTC [4158] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.839 UTC [4158] LOG: entering standby mode logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.864 UTC [4158] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.189 UTC [4158] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.220 UTC [4158] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.222 UTC [4153] LOG: startup process (PID 4158) exited with exit code 1 logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.222 UTC [4153] LOG: aborting startup due to startup process failure logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.222 UTC [4153] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:14 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.230 UTC [4153] LOG: database system is shut down logger.go:42: 19:36:15 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:50.262 UTC [4013] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:50.274 UTC [4013] LOG: database system is shut down logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.817 UTC [4057] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.818 UTC [4057] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.818 UTC [4057] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.844 UTC [4057] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.851 UTC [4062] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.879 UTC [4062] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.906 UTC [4062] LOG: entering standby mode logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:35:59.932 UTC [4062] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.254 UTC [4062] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.291 UTC [4062] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.294 UTC [4057] LOG: startup process (PID 4062) exited with exit code 1 logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.294 UTC [4057] LOG: aborting startup due to startup process failure logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.294 UTC [4057] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:00.307 UTC [4057] LOG: database system is shut down logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.809 UTC [4101] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.809 UTC [4101] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.809 UTC [4101] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.834 UTC [4101] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.841 UTC [4106] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.867 UTC [4106] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.894 UTC [4106] LOG: entering standby mode logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:09.919 UTC [4106] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.239 UTC [4106] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.274 UTC [4106] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.276 UTC [4101] LOG: startup process (PID 4106) exited with exit code 1 logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.276 UTC [4101] LOG: aborting startup due to startup process failure logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.276 UTC [4101] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:16 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.290 UTC [4101] LOG: database system is shut down logger.go:42: 19:36:47 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 22m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 22m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 23m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 19m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 19m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 18m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 17m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 17m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 22m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 22m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 22m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 21m logger.go:42: 19:36:48 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 24m logger.go:42: 19:36:50 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:36:51 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:36:52 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.222 UTC [4153] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.230 UTC [4153] LOG: database system is shut down logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.829 UTC [4230] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.829 UTC [4230] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.829 UTC [4230] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.853 UTC [4230] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.860 UTC [4235] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.884 UTC [4235] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.907 UTC [4235] LOG: entering standby mode logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.930 UTC [4235] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.249 UTC [4235] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.281 UTC [4235] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.282 UTC [4230] LOG: startup process (PID 4235) exited with exit code 1 logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.282 UTC [4230] LOG: aborting startup due to startup process failure logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.282 UTC [4230] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.290 UTC [4230] LOG: database system is shut down logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.751 UTC [4299] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.751 UTC [4299] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.751 UTC [4299] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.775 UTC [4299] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.781 UTC [4304] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.804 UTC [4304] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.828 UTC [4304] LOG: entering standby mode logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.852 UTC [4304] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.170 UTC [4304] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.203 UTC [4304] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.205 UTC [4299] LOG: startup process (PID 4304) exited with exit code 1 logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.205 UTC [4299] LOG: aborting startup due to startup process failure logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.205 UTC [4299] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:53 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.213 UTC [4299] LOG: database system is shut down logger.go:42: 19:36:54 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.276 UTC [4101] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:10.290 UTC [4101] LOG: database system is shut down logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.903 UTC [4178] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.904 UTC [4178] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.904 UTC [4178] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.928 UTC [4178] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.935 UTC [4183] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.964 UTC [4183] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:29.991 UTC [4183] LOG: entering standby mode logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.019 UTC [4183] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.340 UTC [4183] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.376 UTC [4183] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.377 UTC [4178] LOG: startup process (PID 4183) exited with exit code 1 logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.377 UTC [4178] LOG: aborting startup due to startup process failure logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.377 UTC [4178] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:30.388 UTC [4178] LOG: database system is shut down logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.817 UTC [4247] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.818 UTC [4247] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.818 UTC [4247] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.842 UTC [4247] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.850 UTC [4252] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.878 UTC [4252] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.905 UTC [4252] LOG: entering standby mode logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:49.933 UTC [4252] LOG: restored log file "00000003.history" from archive logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.254 UTC [4252] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.290 UTC [4252] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.292 UTC [4247] LOG: startup process (PID 4252) exited with exit code 1 logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.292 UTC [4247] LOG: aborting startup due to startup process failure logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.292 UTC [4247] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:36:55 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:36:50.309 UTC [4247] LOG: database system is shut down logger.go:42: 19:37:27 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 23m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 23m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 23m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 20m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 19m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 18m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 18m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 18m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 22m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 22m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 22m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 21m logger.go:42: 19:37:28 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 25m logger.go:42: 19:37:29 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:37:30 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:37:31 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:00.203 UTC [4351] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:00.212 UTC [4351] LOG: database system is shut down logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.745 UTC [4420] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.746 UTC [4420] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.746 UTC [4420] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.770 UTC [4420] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.776 UTC [4425] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.800 UTC [4425] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.824 UTC [4425] LOG: entering standby mode logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.848 UTC [4425] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.168 UTC [4425] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.201 UTC [4425] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.203 UTC [4420] LOG: startup process (PID 4425) exited with exit code 1 logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.203 UTC [4420] LOG: aborting startup due to startup process failure logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.203 UTC [4420] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.211 UTC [4420] LOG: database system is shut down logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.805 UTC [4465] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.806 UTC [4465] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.806 UTC [4465] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.831 UTC [4465] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.837 UTC [4470] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.863 UTC [4470] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.888 UTC [4470] LOG: entering standby mode logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.914 UTC [4470] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.234 UTC [4470] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.266 UTC [4470] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.268 UTC [4465] LOG: startup process (PID 4470) exited with exit code 1 logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.268 UTC [4465] LOG: aborting startup due to startup process failure logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.268 UTC [4465] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:32 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.276 UTC [4465] LOG: database system is shut down logger.go:42: 19:37:33 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:00.306 UTC [4299] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:00.317 UTC [4299] LOG: database system is shut down logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.811 UTC [4377] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.812 UTC [4377] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.812 UTC [4377] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.836 UTC [4377] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.843 UTC [4382] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.869 UTC [4382] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.896 UTC [4382] LOG: entering standby mode logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:19.924 UTC [4382] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.245 UTC [4382] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.280 UTC [4382] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.284 UTC [4377] LOG: startup process (PID 4382) exited with exit code 1 logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.284 UTC [4377] LOG: aborting startup due to startup process failure logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.284 UTC [4377] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:20.296 UTC [4377] LOG: database system is shut down logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.879 UTC [4421] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.879 UTC [4421] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.879 UTC [4421] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.904 UTC [4421] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.911 UTC [4426] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.940 UTC [4426] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.967 UTC [4426] LOG: entering standby mode logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:29.994 UTC [4426] LOG: restored log file "00000003.history" from archive logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.316 UTC [4426] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.351 UTC [4426] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.353 UTC [4421] LOG: startup process (PID 4426) exited with exit code 1 logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.353 UTC [4421] LOG: aborting startup due to startup process failure logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.353 UTC [4421] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:37:34 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.366 UTC [4421] LOG: database system is shut down logger.go:42: 19:38:06 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 23m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 23m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 24m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 21m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 20m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 19m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 18m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 18m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 23m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 23m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 23m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 22m logger.go:42: 19:38:07 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 26m logger.go:42: 19:38:08 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:38:09 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:38:10 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.268 UTC [4465] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.276 UTC [4465] LOG: database system is shut down logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.747 UTC [4551] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.748 UTC [4551] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.748 UTC [4551] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.772 UTC [4551] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.778 UTC [4556] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.803 UTC [4556] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.826 UTC [4556] LOG: entering standby mode logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.849 UTC [4556] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.169 UTC [4556] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.201 UTC [4556] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.203 UTC [4551] LOG: startup process (PID 4556) exited with exit code 1 logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.203 UTC [4551] LOG: aborting startup due to startup process failure logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.203 UTC [4551] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.211 UTC [4551] LOG: database system is shut down logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.756 UTC [4595] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.757 UTC [4595] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.757 UTC [4595] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.781 UTC [4595] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.787 UTC [4600] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.811 UTC [4600] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.834 UTC [4600] LOG: entering standby mode logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.858 UTC [4600] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.177 UTC [4600] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.210 UTC [4600] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.211 UTC [4595] LOG: startup process (PID 4600) exited with exit code 1 logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.211 UTC [4595] LOG: aborting startup due to startup process failure logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.211 UTC [4595] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:11 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.220 UTC [4595] LOG: database system is shut down logger.go:42: 19:38:12 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.353 UTC [4421] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:30.366 UTC [4421] LOG: database system is shut down logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.841 UTC [4498] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.842 UTC [4498] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.842 UTC [4498] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.866 UTC [4498] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.874 UTC [4503] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.902 UTC [4503] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.927 UTC [4503] LOG: entering standby mode logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:49.952 UTC [4503] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.275 UTC [4503] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.310 UTC [4503] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.312 UTC [4498] LOG: startup process (PID 4503) exited with exit code 1 logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.312 UTC [4498] LOG: aborting startup due to startup process failure logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.312 UTC [4498] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:50.323 UTC [4498] LOG: database system is shut down logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.800 UTC [4542] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.801 UTC [4542] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.801 UTC [4542] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.826 UTC [4542] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.832 UTC [4547] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.859 UTC [4547] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.887 UTC [4547] LOG: entering standby mode logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:37:59.912 UTC [4547] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.234 UTC [4547] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.270 UTC [4547] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.272 UTC [4542] LOG: startup process (PID 4547) exited with exit code 1 logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.272 UTC [4542] LOG: aborting startup due to startup process failure logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.272 UTC [4542] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:13 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:00.287 UTC [4542] LOG: database system is shut down logger.go:42: 19:38:45 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 24m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 24m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 24m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 21m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 21m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 20m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 19m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 19m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 24m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 24m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 24m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 23m logger.go:42: 19:38:46 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 26m logger.go:42: 19:38:47 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:38:48 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:38:49 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:20.206 UTC [4671] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:20.214 UTC [4671] LOG: database system is shut down logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.799 UTC [4715] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.799 UTC [4715] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.799 UTC [4715] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.823 UTC [4715] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.830 UTC [4720] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.855 UTC [4720] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.878 UTC [4720] LOG: entering standby mode logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.902 UTC [4720] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.220 UTC [4720] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.254 UTC [4720] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.255 UTC [4715] LOG: startup process (PID 4720) exited with exit code 1 logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.255 UTC [4715] LOG: aborting startup due to startup process failure logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.255 UTC [4715] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.263 UTC [4715] LOG: database system is shut down logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.750 UTC [4784] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.750 UTC [4784] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.750 UTC [4784] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.775 UTC [4784] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.782 UTC [4789] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.806 UTC [4789] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.829 UTC [4789] LOG: entering standby mode logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.854 UTC [4789] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.173 UTC [4789] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.206 UTC [4789] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.207 UTC [4784] LOG: startup process (PID 4789) exited with exit code 1 logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.207 UTC [4784] LOG: aborting startup due to startup process failure logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.207 UTC [4784] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:50 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.215 UTC [4784] LOG: database system is shut down logger.go:42: 19:38:51 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:20.292 UTC [4618] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:20.305 UTC [4618] LOG: database system is shut down logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.873 UTC [4671] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.874 UTC [4671] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.874 UTC [4671] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.899 UTC [4671] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.906 UTC [4676] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.934 UTC [4676] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.961 UTC [4676] LOG: entering standby mode logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:29.988 UTC [4676] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.310 UTC [4676] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.352 UTC [4676] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.354 UTC [4671] LOG: startup process (PID 4676) exited with exit code 1 logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.354 UTC [4671] LOG: aborting startup due to startup process failure logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.354 UTC [4671] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:30.368 UTC [4671] LOG: database system is shut down logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.795 UTC [4741] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.796 UTC [4741] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.796 UTC [4741] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.821 UTC [4741] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.828 UTC [4746] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.855 UTC [4746] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.881 UTC [4746] LOG: entering standby mode logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:49.910 UTC [4746] LOG: restored log file "00000003.history" from archive logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.231 UTC [4746] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.266 UTC [4746] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.270 UTC [4741] LOG: startup process (PID 4746) exited with exit code 1 logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.270 UTC [4741] LOG: aborting startup due to startup process failure logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.270 UTC [4741] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:38:52 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.283 UTC [4741] LOG: database system is shut down logger.go:42: 19:39:24 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 25m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 25m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 25m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 22m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 21m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 20m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 20m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 20m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 24m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 24m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 24m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 23m logger.go:42: 19:39:25 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 27m logger.go:42: 19:39:26 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:39:27 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:39:28 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.207 UTC [4784] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:50.215 UTC [4784] LOG: database system is shut down logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:59.752 UTC [4836] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:59.752 UTC [4836] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:59.752 UTC [4836] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:59.776 UTC [4836] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:59.783 UTC [4841] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:59.809 UTC [4841] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:59.834 UTC [4841] LOG: entering standby mode logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:38:59.857 UTC [4841] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:00.176 UTC [4841] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:00.209 UTC [4841] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:00.210 UTC [4836] LOG: startup process (PID 4841) exited with exit code 1 logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:00.210 UTC [4836] LOG: aborting startup due to startup process failure logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:00.210 UTC [4836] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:00.218 UTC [4836] LOG: database system is shut down logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:09.761 UTC [4880] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:09.761 UTC [4880] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:09.761 UTC [4880] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:09.785 UTC [4880] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:09.791 UTC [4885] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:09.816 UTC [4885] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:09.840 UTC [4885] LOG: entering standby mode logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:09.865 UTC [4885] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:10.184 UTC [4885] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:10.216 UTC [4885] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:10.217 UTC [4880] LOG: startup process (PID 4885) exited with exit code 1 logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:10.217 UTC [4880] LOG: aborting startup due to startup process failure logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:10.217 UTC [4880] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:29 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:10.225 UTC [4880] LOG: database system is shut down logger.go:42: 19:39:30 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:00.260 UTC [4793] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:00.271 UTC [4793] LOG: database system is shut down logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:19.839 UTC [4861] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:19.839 UTC [4861] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:19.839 UTC [4861] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:19.865 UTC [4861] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:19.872 UTC [4866] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:19.900 UTC [4866] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:19.926 UTC [4866] LOG: entering standby mode logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:19.951 UTC [4866] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:20.272 UTC [4866] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:20.308 UTC [4866] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:20.311 UTC [4861] LOG: startup process (PID 4866) exited with exit code 1 logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:20.311 UTC [4861] LOG: aborting startup due to startup process failure logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:20.311 UTC [4861] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:20.325 UTC [4861] LOG: database system is shut down logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.907 UTC [4914] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.908 UTC [4914] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.908 UTC [4914] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.932 UTC [4914] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.939 UTC [4919] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.966 UTC [4919] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.991 UTC [4919] LOG: entering standby mode logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.018 UTC [4919] LOG: restored log file "00000003.history" from archive logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.339 UTC [4919] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.377 UTC [4919] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.379 UTC [4914] LOG: startup process (PID 4919) exited with exit code 1 logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.379 UTC [4914] LOG: aborting startup due to startup process failure logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.379 UTC [4914] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:39:31 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.388 UTC [4914] LOG: database system is shut down logger.go:42: 19:40:03 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 25m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 25m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 26m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 23m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 22m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 21m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 20m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 20m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 25m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 25m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 25m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 24m logger.go:42: 19:40:04 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 28m logger.go:42: 19:40:05 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:40:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:40:07 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:10.217 UTC [4880] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:10.225 UTC [4880] LOG: database system is shut down logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.917 UTC [4949] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.917 UTC [4949] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.917 UTC [4949] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.941 UTC [4949] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.948 UTC [4962] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.973 UTC [4962] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:29.997 UTC [4962] LOG: entering standby mode logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.021 UTC [4962] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.340 UTC [4962] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.372 UTC [4962] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.374 UTC [4949] LOG: startup process (PID 4962) exited with exit code 1 logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.374 UTC [4949] LOG: aborting startup due to startup process failure logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.374 UTC [4949] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.382 UTC [4949] LOG: database system is shut down logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.754 UTC [5026] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.755 UTC [5026] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.755 UTC [5026] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.779 UTC [5026] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.786 UTC [5031] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.810 UTC [5031] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.833 UTC [5031] LOG: entering standby mode logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.862 UTC [5031] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.182 UTC [5031] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.215 UTC [5031] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.217 UTC [5026] LOG: startup process (PID 5031) exited with exit code 1 logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.217 UTC [5026] LOG: aborting startup due to startup process failure logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.217 UTC [5026] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.225 UTC [5026] LOG: database system is shut down logger.go:42: 19:40:09 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.379 UTC [4914] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:30.388 UTC [4914] LOG: database system is shut down logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.817 UTC [4991] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.818 UTC [4991] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.818 UTC [4991] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.843 UTC [4991] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.850 UTC [4996] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.878 UTC [4996] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.905 UTC [4996] LOG: entering standby mode logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:49.932 UTC [4996] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.254 UTC [4996] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.288 UTC [4996] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.290 UTC [4991] LOG: startup process (PID 4996) exited with exit code 1 logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.290 UTC [4991] LOG: aborting startup due to startup process failure logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.290 UTC [4991] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:39:50.302 UTC [4991] LOG: database system is shut down logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:09.840 UTC [5058] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:09.840 UTC [5058] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:09.840 UTC [5058] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:09.865 UTC [5058] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:09.872 UTC [5065] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:09.901 UTC [5065] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:09.928 UTC [5065] LOG: entering standby mode logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:09.956 UTC [5065] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:10.279 UTC [5065] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:10.314 UTC [5065] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:10.316 UTC [5058] LOG: startup process (PID 5065) exited with exit code 1 logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:10.316 UTC [5058] LOG: aborting startup due to startup process failure logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:10.316 UTC [5058] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:10 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:10.330 UTC [5058] LOG: database system is shut down logger.go:42: 19:40:42 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 26m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 26m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 26m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 23m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 23m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 22m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 21m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 21m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 26m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 26m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 26m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 24m logger.go:42: 19:40:43 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 28m logger.go:42: 19:40:44 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:40:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:40:46 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:10.216 UTC [5103] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:10.225 UTC [5103] LOG: database system is shut down logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.765 UTC [5146] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.765 UTC [5146] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.765 UTC [5146] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.790 UTC [5146] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.796 UTC [5151] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.821 UTC [5151] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.845 UTC [5151] LOG: entering standby mode logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.869 UTC [5151] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.189 UTC [5151] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.222 UTC [5151] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.223 UTC [5146] LOG: startup process (PID 5151) exited with exit code 1 logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.223 UTC [5146] LOG: aborting startup due to startup process failure logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.223 UTC [5146] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.232 UTC [5146] LOG: database system is shut down logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.835 UTC [5199] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.836 UTC [5199] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.836 UTC [5199] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.861 UTC [5199] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.867 UTC [5204] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.892 UTC [5204] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.916 UTC [5204] LOG: entering standby mode logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.941 UTC [5204] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.260 UTC [5204] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.293 UTC [5204] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.294 UTC [5199] LOG: startup process (PID 5204) exited with exit code 1 logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.294 UTC [5199] LOG: aborting startup due to startup process failure logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.294 UTC [5199] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.302 UTC [5199] LOG: database system is shut down logger.go:42: 19:40:48 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.824 UTC [5110] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.831 UTC [5117] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.858 UTC [5117] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.886 UTC [5117] LOG: entering standby mode logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:19.913 UTC [5117] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.235 UTC [5117] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.270 UTC [5117] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.272 UTC [5110] LOG: startup process (PID 5117) exited with exit code 1 logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.272 UTC [5110] LOG: aborting startup due to startup process failure logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.272 UTC [5110] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:20.287 UTC [5110] LOG: database system is shut down logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.919 UTC [5156] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.920 UTC [5156] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.920 UTC [5156] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.945 UTC [5156] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.952 UTC [5161] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:29.982 UTC [5161] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.009 UTC [5161] LOG: entering standby mode logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.037 UTC [5161] LOG: restored log file "00000003.history" from archive logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.358 UTC [5161] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.393 UTC [5161] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.395 UTC [5156] LOG: startup process (PID 5161) exited with exit code 1 logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.395 UTC [5156] LOG: aborting startup due to startup process failure logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.395 UTC [5156] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:30.404 UTC [5156] LOG: database system is shut down logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:49.798 UTC [5225] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:49.799 UTC [5225] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:49.799 UTC [5225] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:49.830 UTC [5225] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:40:49 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:49.837 UTC [5237] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:41:21 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 27m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 27m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 27m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 24m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 23m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 22m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 22m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 22m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 26m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 26m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 26m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 25m logger.go:42: 19:41:22 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 29m logger.go:42: 19:41:23 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:41:24 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:41:25 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:50.219 UTC [5276] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:50.227 UTC [5276] LOG: database system is shut down logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.757 UTC [5320] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.758 UTC [5320] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.758 UTC [5320] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.782 UTC [5320] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.789 UTC [5325] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.814 UTC [5325] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.838 UTC [5325] LOG: entering standby mode logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.861 UTC [5325] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.180 UTC [5325] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.212 UTC [5325] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.214 UTC [5320] LOG: startup process (PID 5325) exited with exit code 1 logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.214 UTC [5320] LOG: aborting startup due to startup process failure logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.214 UTC [5320] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.222 UTC [5320] LOG: database system is shut down logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.750 UTC [5391] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.751 UTC [5391] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.751 UTC [5391] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.775 UTC [5391] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.782 UTC [5396] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.806 UTC [5396] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.828 UTC [5396] LOG: entering standby mode logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.852 UTC [5396] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.171 UTC [5396] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.203 UTC [5396] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.205 UTC [5391] LOG: startup process (PID 5396) exited with exit code 1 logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.205 UTC [5391] LOG: aborting startup due to startup process failure logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.205 UTC [5391] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:26 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.213 UTC [5391] LOG: database system is shut down logger.go:42: 19:41:27 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:50.282 UTC [5225] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:50.296 UTC [5225] LOG: database system is shut down logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.784 UTC [5278] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.784 UTC [5278] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.784 UTC [5278] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.809 UTC [5278] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.816 UTC [5283] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.843 UTC [5283] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.868 UTC [5283] LOG: entering standby mode logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:40:59.894 UTC [5283] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.215 UTC [5283] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.248 UTC [5283] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.250 UTC [5278] LOG: startup process (PID 5283) exited with exit code 1 logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.250 UTC [5278] LOG: aborting startup due to startup process failure logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.250 UTC [5278] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:00.258 UTC [5278] LOG: database system is shut down logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.832 UTC [5347] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.833 UTC [5347] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.833 UTC [5347] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.858 UTC [5347] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.866 UTC [5352] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.896 UTC [5352] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.924 UTC [5352] LOG: entering standby mode logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:19.952 UTC [5352] LOG: restored log file "00000003.history" from archive logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.274 UTC [5352] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.313 UTC [5352] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.316 UTC [5347] LOG: startup process (PID 5352) exited with exit code 1 logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.316 UTC [5347] LOG: aborting startup due to startup process failure logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.316 UTC [5347] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:41:28 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.331 UTC [5347] LOG: database system is shut down logger.go:42: 19:42:00 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 27m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 27m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 28m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 25m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 24m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 23m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 22m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 22m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 27m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 27m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 27m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 26m logger.go:42: 19:42:01 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 30m logger.go:42: 19:42:02 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:42:03 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:42:04 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.205 UTC [5391] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:20.213 UTC [5391] LOG: database system is shut down logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:29.791 UTC [5443] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:29.791 UTC [5443] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:29.791 UTC [5443] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:29.816 UTC [5443] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:29.822 UTC [5448] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:29.846 UTC [5448] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:29.869 UTC [5448] LOG: entering standby mode logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:29.893 UTC [5448] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:30.211 UTC [5448] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:30.243 UTC [5448] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:30.245 UTC [5443] LOG: startup process (PID 5448) exited with exit code 1 logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:30.245 UTC [5443] LOG: aborting startup due to startup process failure logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:30.245 UTC [5443] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:30.254 UTC [5443] LOG: database system is shut down logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.743 UTC [5512] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.744 UTC [5512] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.744 UTC [5512] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.769 UTC [5512] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.775 UTC [5517] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.800 UTC [5517] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.824 UTC [5517] LOG: entering standby mode logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.848 UTC [5517] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.167 UTC [5517] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.199 UTC [5517] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.201 UTC [5512] LOG: startup process (PID 5517) exited with exit code 1 logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.201 UTC [5512] LOG: aborting startup due to startup process failure logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.201 UTC [5512] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.209 UTC [5512] LOG: database system is shut down logger.go:42: 19:42:06 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:30.354 UTC [5399] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:30.369 UTC [5399] LOG: database system is shut down logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.814 UTC [5468] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.814 UTC [5468] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.814 UTC [5468] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.839 UTC [5468] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.846 UTC [5473] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.873 UTC [5473] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.899 UTC [5473] LOG: entering standby mode logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:49.927 UTC [5473] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.249 UTC [5473] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.284 UTC [5473] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.287 UTC [5468] LOG: startup process (PID 5473) exited with exit code 1 logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.287 UTC [5468] LOG: aborting startup due to startup process failure logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.287 UTC [5468] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.299 UTC [5468] LOG: database system is shut down logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:59.818 UTC [5512] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:59.819 UTC [5512] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:59.819 UTC [5512] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:59.844 UTC [5512] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:59.851 UTC [5517] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:59.880 UTC [5517] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:59.907 UTC [5517] LOG: entering standby mode logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:59.934 UTC [5517] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:00.255 UTC [5517] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:00.291 UTC [5517] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:00.293 UTC [5512] LOG: startup process (PID 5517) exited with exit code 1 logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:00.293 UTC [5512] LOG: aborting startup due to startup process failure logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:00.293 UTC [5512] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:08 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:00.305 UTC [5512] LOG: database system is shut down logger.go:42: 19:42:39 | major-upgrade-14-to-15/10-run-restore | running command: [sh -c set -o errexit kubectl -n ${NAMESPACE} get pod for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi echo "PostgreSQL logs from ${pod}:" echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done sleep 30] logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | NAME READY STATUS RESTARTS AGE logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 0/1 Completed 0 28m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 0/1 Completed 0 28m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | 14-to-15-pgdata-rcn2d 0/1 Completed 0 28m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-d4s9-sjqsq 0/1 Completed 0 25m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-backup-vs9w-rrwh5 0/1 Completed 0 25m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-rvwt-0 4/4 Running 0 24m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-s7px-0 3/4 Running 0 23m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-instance1-w64d-0 3/4 Running 0 23m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 2/2 Running 0 28m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 2/2 Running 0 28m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 2/2 Running 0 28m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | major-upgrade-14-to-15-repo-host-0 2/2 Running 0 26m logger.go:42: 19:42:40 | major-upgrade-14-to-15/10-run-restore | pg-client-65d98588cc-pgz5b 1/1 Running 0 30m logger.go:42: 19:42:41 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-rvwt-0: logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.011 UTC [134] LOG: checkpoint complete: wrote 5 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.208 s, sync=0.003 s, total=0.221 s; sync files=5, longest=0.002 s, average=0.001 s; distance=0 kB, estimate=0 kB logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:52.039 P00 ERROR: [103]: unable to find a valid repository: logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | repo1: [ArchiveMismatchError] PostgreSQL version 15, system-id 7609027813903401056 do not match repo1 stanza version 14, system-id 7609027402155872385 logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | HINT: are you archiving to the correct stanza? logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:58.226 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:14:59.079 UTC [134] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.803 s, sync=0.003 s, total=0.854 s; sync files=5, longest=0.002 s, average=0.001 s; distance=16383 kB, estimate=16383 kB logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000B.00000028.backup": read 371 of 8192 logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.397 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:15:11.460 UTC [134] LOG: checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.063 s; sync files=0, longest=0.000 s, average=0.000 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000D.00000028.backup": read 338 of 8192 logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:16:58.661 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.740 UTC [134] LOG: checkpoint complete: wrote 42 buffers (0.3%); 0 WAL file(s) added, 0 removed, 0 recycled; write=4.014 s, sync=0.004 s, total=4.080 s; sync files=13, longest=0.003 s, average=0.001 s; distance=32768 kB, estimate=32768 kB logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] LOG: restore point "pgBackRest Archive Check" created at 0/F000140 logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:02.742 UTC [1209] STATEMENT: select pg_catalog.pg_create_restore_point('pgBackRest Archive Check')::text logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:11.581 P00 ERROR: [082]: unable to push WAL file '00000002000000000000000E' to the archive asynchronously after 1m second(s) logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | HINT: check '/pgdata/pgbackrest/log/db-archive-push-async.log' for errors. logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000002000000000000000F.00000028.backup": read 372 of 8192 logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:34.608 UTC [134] LOG: checkpoint starting: force wait logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:17:35.058 UTC [134] LOG: checkpoint complete: wrote 6 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.404 s, sync=0.002 s, total=0.451 s; sync files=6, longest=0.001 s, average=0.001 s; distance=49152 kB, estimate=49152 kB logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.025 UTC [123] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.026 UTC [123] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.049 UTC [123] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.058 UTC [128] LOG: database system was shut down at 2026-02-20 19:18:33 UTC logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.080 UTC [123] LOG: database system is ready to accept connections logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.141 UTC [132] LOG: pg_cron scheduler started logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | pg_waldump: error: could not read file "00000003.history": read 33 of 8192 logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:19:07.998 UTC [123] LOG: received SIGHUP, reloading configuration files logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:07.218 UTC [126] LOG: checkpoint starting: time logger.go:42: 19:42:42 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:24:10.547 UTC [126] LOG: checkpoint complete: wrote 37 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.314 s, sync=0.005 s, total=3.330 s; sync files=32, longest=0.002 s, average=0.001 s; distance=16384 kB, estimate=16384 kB logger.go:42: 19:42:43 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-s7px-0: logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.201 UTC [5512] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:41:50.209 UTC [5512] LOG: database system is shut down logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:09.757 UTC [5589] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:09.758 UTC [5589] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:09.758 UTC [5589] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:09.783 UTC [5589] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:09.789 UTC [5594] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:09.814 UTC [5594] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:09.837 UTC [5594] LOG: entering standby mode logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:09.862 UTC [5594] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:10.181 UTC [5594] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:10.213 UTC [5594] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:10.215 UTC [5589] LOG: startup process (PID 5594) exited with exit code 1 logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:10.215 UTC [5589] LOG: aborting startup due to startup process failure logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:10.215 UTC [5589] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:10.223 UTC [5589] LOG: database system is shut down logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.795 UTC [5658] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.796 UTC [5658] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.796 UTC [5658] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.821 UTC [5658] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.827 UTC [5663] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.858 UTC [5663] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.882 UTC [5663] LOG: entering standby mode logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.909 UTC [5663] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.229 UTC [5663] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.262 UTC [5663] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.263 UTC [5658] LOG: startup process (PID 5663) exited with exit code 1 logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.263 UTC [5658] LOG: aborting startup due to startup process failure logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.263 UTC [5658] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.271 UTC [5658] LOG: database system is shut down logger.go:42: 19:42:45 | major-upgrade-14-to-15/10-run-restore | PostgreSQL logs from major-upgrade-14-to-15-instance1-w64d-0: logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:00.293 UTC [5512] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:00.305 UTC [5512] LOG: database system is shut down logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:19.773 UTC [5597] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:19.773 UTC [5597] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:19.773 UTC [5597] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:19.798 UTC [5597] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:19.805 UTC [5602] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:19.836 UTC [5602] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:19.861 UTC [5602] LOG: entering standby mode logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:19.887 UTC [5602] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:20.207 UTC [5602] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:20.241 UTC [5602] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:20.242 UTC [5597] LOG: startup process (PID 5602) exited with exit code 1 logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:20.242 UTC [5597] LOG: aborting startup due to startup process failure logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:20.242 UTC [5597] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:20.252 UTC [5597] LOG: database system is shut down logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.861 UTC [5641] LOG: starting PostgreSQL 15.15 - Percona Distribution on x86_64-pc-linux-gnu, compiled by gcc (GCC) 11.5.0 20240719 (Red Hat 11.5.0-11.0.1), 64-bit logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.861 UTC [5641] LOG: listening on IPv4 address "0.0.0.0", port 5432 logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.862 UTC [5641] LOG: listening on IPv6 address "::", port 5432 logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.886 UTC [5641] LOG: listening on Unix socket "/tmp/postgres/.s.PGSQL.5432" logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.893 UTC [5646] LOG: database system was shut down in recovery at 2026-02-20 19:18:06 UTC logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.921 UTC [5646] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.948 UTC [5646] LOG: entering standby mode logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:29.975 UTC [5646] LOG: restored log file "00000003.history" from archive logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.299 UTC [5646] LOG: restored log file "000000030000000000000012" from archive logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.347 UTC [5646] FATAL: requested timeline 3 does not contain minimum recovery point 0/13006810 on timeline 2 logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.350 UTC [5641] LOG: startup process (PID 5646) exited with exit code 1 logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.350 UTC [5641] LOG: aborting startup due to startup process failure logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.350 UTC [5641] LOG: [pg_stat_monitor] pgsm_shmem_shutdown: Shutdown initiated. logger.go:42: 19:42:47 | major-upgrade-14-to-15/10-run-restore | 2026-02-20 19:42:30.363 UTC [5641] LOG: database system is shut down logger.go:42: 19:43:03 | major-upgrade-14-to-15/10-run-restore | test step failed 10-run-restore logger.go:42: 19:43:03 | major-upgrade-14-to-15/10-run-restore | collecting log output for [type==pod,label: postgres-operator.crunchydata.com/data=postgres] logger.go:42: 19:43:03 | major-upgrade-14-to-15/10-run-restore | running command: [kubectl logs --prefix -l postgres-operator.crunchydata.com/data=postgres -n kuttl-test-safe-skylark --all-containers --tail=30] logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + IFS=, logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + read -ra extensions logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/15') logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + declare -a args logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + [[ -n '' ]] logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + [[ '' == \t\r\u\e ]] logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + [[ '' == \t\r\u\e ]] logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] ++ basename -- /pgdata/extension/15/pg_cron-pg15-1.6.6.installed logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + filename=pg_cron-pg15-1.6.6.installed logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + key=pg_cron-pg15-1.6.6 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + [[ pg_cron-pg15-1.6.6 == \* ]] logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + [[ ! pg_cron-pg15-1.6.6 =~ pg_cron-pg15-1.6.6 ]] logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + for key in "${extensions[@]}" logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + '[' -f /pgdata/extension/15/pg_cron-pg15-1.6.6.installed ']' logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + echo 'Extension pg_cron-pg15-1.6.6 already installed' logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] + continue logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/extension-installer-15] Extension pg_cron-pg15-1.6.6 already installed logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=105-e92b0f15 --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@607a496c-aa85-48b9-a679-46c165eb51f8=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] max_connections setting: 100 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] max_worker_processes setting: 8 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] max_wal_senders setting: 10 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] max_prepared_xacts setting: 0 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] max_locks_per_xact setting: 64 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] track_commit_timestamp setting: on logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Maximum data alignment: 8 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Database block size: 8192 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Blocks per segment of large relation: 131072 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] WAL block size: 8192 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Bytes per WAL segment: 16777216 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Maximum length of identifiers: 64 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Maximum columns in an index: 32 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Maximum size of a TOAST chunk: 1996 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Size of a large-object chunk: 2048 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Date/time type storage: 64-bit integers logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Float8 argument passing: by value logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Data page checksum version: 1 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] Mock authentication nonce: c576dec749478ad233f5474bc07dfe73cb992e7e8585bfda8c76f1016c3e41dc logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:49,041 INFO: Lock owner: major-upgrade-14-to-15-instance1-rvwt-0; I am major-upgrade-14-to-15-instance1-s7px-0 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:49,043 INFO: starting as a secondary logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:49.524 UTC [5735] LOG: pgaudit extension initialized logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:49,537 INFO: postmaster pid=5735 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] /tmp/postgres:5432 - no response logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:49.750 UTC [5735] LOG: redirecting log output to logging collector process logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:49.750 UTC [5735] HINT: Future log output will appear in directory "log". logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:50,549 ERROR: postmaster is not running logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:59,030 INFO: Lock owner: major-upgrade-14-to-15-instance1-rvwt-0; I am major-upgrade-14-to-15-instance1-s7px-0 logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database] 2026-02-20 19:42:59,030 INFO: failed to start postgres logger.go:42: 19:43:04 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=113-77e2e86c --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@607a496c-aa85-48b9-a679-46c165eb51f8=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] + CRUNCHY_BINDIR=/opt/crunchy logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/pgbackrest /opt/crunchy/bin/pgbackrest logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-entrypoint.sh /opt/crunchy/bin/postgres-entrypoint.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-liveness-check.sh /opt/crunchy/bin/postgres-liveness-check.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/postgres-readiness-check.sh /opt/crunchy/bin/postgres-readiness-check.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/relocate-extensions.sh /opt/crunchy/bin/relocate-extensions.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -u logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] ++ id -g logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/database-init] + install -o 2 -g 2 -m 0755 -D /usr/local/bin/restore-command-wrapper.sh /opt/crunchy/bin/restore-command-wrapper.sh logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] Initializing ... logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-15/bin/postgres logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 15.15 - Percona Distribution logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg15 logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg15 logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: data version::15 logger.go:42: 19:43:05 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg15_wal logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + IFS=, logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + read -ra extensions logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/15') logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + declare -a args logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + [[ -n '' ]] logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + [[ '' == \t\r\u\e ]] logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + [[ '' == \t\r\u\e ]] logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] ++ basename -- /pgdata/extension/15/pg_cron-pg15-1.6.6.installed logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + filename=pg_cron-pg15-1.6.6.installed logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + key=pg_cron-pg15-1.6.6 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + [[ pg_cron-pg15-1.6.6 == \* ]] logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + [[ ! pg_cron-pg15-1.6.6 =~ pg_cron-pg15-1.6.6 ]] logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + for key in "${extensions[@]}" logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + '[' -f /pgdata/extension/15/pg_cron-pg15-1.6.6.installed ']' logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + echo 'Extension pg_cron-pg15-1.6.6 already installed' logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] + continue logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/extension-installer-15] Extension pg_cron-pg15-1.6.6 already installed logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] Initializing ... logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-15/bin/postgres logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 15.15 - Percona Distribution logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg15 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg15 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: data version::15 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg15_wal logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] max_connections setting: 100 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] max_worker_processes setting: 8 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] max_wal_senders setting: 10 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] max_prepared_xacts setting: 0 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] max_locks_per_xact setting: 64 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] track_commit_timestamp setting: on logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Maximum data alignment: 8 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Database block size: 8192 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Blocks per segment of large relation: 131072 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] WAL block size: 8192 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Bytes per WAL segment: 16777216 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Maximum length of identifiers: 64 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Maximum columns in an index: 32 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Maximum size of a TOAST chunk: 1996 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Size of a large-object chunk: 2048 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Date/time type storage: 64-bit integers logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Float8 argument passing: by value logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Data page checksum version: 1 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] Mock authentication nonce: c576dec749478ad233f5474bc07dfe73cb992e7e8585bfda8c76f1016c3e41dc logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:49,043 INFO: Lock owner: major-upgrade-14-to-15-instance1-rvwt-0; I am major-upgrade-14-to-15-instance1-w64d-0 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:49,044 INFO: starting as a secondary logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:49.547 UTC [5718] LOG: pgaudit extension initialized logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:49,561 INFO: postmaster pid=5718 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] /tmp/postgres:5432 - no response logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:49.821 UTC [5718] LOG: redirecting log output to logging collector process logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:49.821 UTC [5718] HINT: Future log output will appear in directory "log". logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:50,575 ERROR: postmaster is not running logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:59,030 INFO: Lock owner: major-upgrade-14-to-15-instance1-rvwt-0; I am major-upgrade-14-to-15-instance1-w64d-0 logger.go:42: 19:43:06 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/database] 2026-02-20 19:42:59,031 INFO: failed to start postgres logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] Initializing ... logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: uid::26 logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: gid::26 1000 logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: postgres path::/usr/pgsql-15/bin/postgres logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: postgres version::postgres (PostgreSQL) 15.15 - Percona Distribution logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: config directory::/pgdata/pg15 logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: data directory::/pgdata/pg15 logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: bootstrap directory::/pgdata/pg15_bootstrap logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: pgBackRest log directory::/pgdata/pgbackrest/log logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: data version::15 logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/postgres-startup] ::postgres-operator: wal directory::/pgdata/pg15_wal logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-s7px-0/pgbackrest] P00 INFO: server command begin 2.57.0: --exec-id=103-ee8d497d --log-level-console=detail --log-level-file=off --log-level-stderr=error --log-path=/pgdata/pgbackrest/log --no-log-timestamp --tls-server-address=0.0.0.0 --tls-server-auth=pgbackrest@607a496c-aa85-48b9-a679-46c165eb51f8=* --tls-server-ca-file=/etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt --tls-server-cert-file=/etc/pgbackrest/server/server-tls.crt --tls-server-key-file=/etc/pgbackrest/server/server-tls.key logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/nss-wrapper-init] nss_wrapper: user exists logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/nss-wrapper-init] nss_wrapper: group exists logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/nss-wrapper-init] nss_wrapper: environment configured logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + IFS=, logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + read -ra extensions logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + args=('-type' 's3' '-region' 'eu-central-1' '-bucket' 'pg-extensions' '-extension-path' '/pgdata/extension/15') logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + declare -a args logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + [[ -n '' ]] logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + [[ '' == \t\r\u\e ]] logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + [[ '' == \t\r\u\e ]] logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + for installed in "${PGDATA_EXTENSIONS}"/*.installed logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] ++ basename -- /pgdata/extension/15/pg_cron-pg15-1.6.6.installed logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + filename=pg_cron-pg15-1.6.6.installed logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + key=pg_cron-pg15-1.6.6 logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + [[ pg_cron-pg15-1.6.6 == \* ]] logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + [[ ! pg_cron-pg15-1.6.6 =~ pg_cron-pg15-1.6.6 ]] logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + for key in "${extensions[@]}" logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + '[' -f /pgdata/extension/15/pg_cron-pg15-1.6.6.installed ']' logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + echo 'Extension pg_cron-pg15-1.6.6 already installed' logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] + continue logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-w64d-0/extension-installer-15] Extension pg_cron-pg15-1.6.6 already installed logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:38:19,020 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:38:29,079 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:38:39,023 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:38:49,020 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:38:59,023 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:39:09,019 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:39:19,021 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:39:29,081 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:39:39,026 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:39:49,023 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:39:59,021 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:40:09,021 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:40:09,348 ERROR: ObjectCache.run ProtocolError("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:40:19,022 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:40:29,103 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:40:39,021 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:40:49,019 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:40:59,025 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:41:09,022 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:41:19,022 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:41:29,068 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:41:39,021 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:41:49,019 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:41:59,022 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:42:09,020 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:42:19,021 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:42:29,071 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:42:39,019 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:42:49,022 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock logger.go:42: 19:43:07 | major-upgrade-14-to-15/10-run-restore | [pod/major-upgrade-14-to-15-instance1-rvwt-0/database] 2026-02-20 19:42:59,019 INFO: no action. I am (major-upgrade-14-to-15-instance1-rvwt-0), the leader with the lock case.go:328: failed in step 10-run-restore case.go:330: --- PerconaPGCluster:kuttl-test-safe-skylark/major-upgrade-14-to-15 +++ PerconaPGCluster:kuttl-test-safe-skylark/major-upgrade-14-to-15 @@ -1,18 +1,41 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"pgv2.percona.com/v2","kind":"PerconaPGCluster","metadata":{"annotations":{},"labels":{"e2e":"major-upgrade-14-to-15"},"name":"major-upgrade-14-to-15","namespace":"kuttl-test-safe-skylark"},"spec":{"backups":{"pgbackrest":{"image":"perconalab/percona-postgresql-operator:main-pgbackrest14","manual":{"options":["--type=full"],"repoName":"repo1"},"repoHost":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/data":"pgbackrest"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}}},"repos":[{"name":"repo1","schedules":{"full":"0 0 * * 6"},"volume":{"volumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}}}}]}},"crVersion":"2.9.0","extensions":{"custom":[{"name":"pg_cron","version":"1.6.6"}],"image":"docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0","imagePullPolicy":"Always","storage":{"bucket":"pg-extensions","region":"eu-central-1","secret":{"name":"aws-s3-secret"},"type":"s3"}},"image":"perconalab/percona-postgresql-operator:main-ppg14-postgres","imagePullPolicy":"Always","initContainer":{"image":"docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0"},"instances":[{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/data":"postgres"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"3Gi"}}},"name":"instance1","replicas":3}],"patroni":{"dynamicConfiguration":{"postgresql":{"parameters":{"shared_preload_libraries":"pg_cron"}}}},"pmm":{"customClusterName":"major-upgrade-14-to-15-pmm-custom-name","enabled":false,"image":"docker.io/perconalab/pmm-client:dev-latest","postgresParams":"--environment=dev-postgres","secret":"major-upgrade-14-to-15-pmm-secret","serverHost":"monitoring-service"},"postgresVersion":14,"proxy":{"pgBouncer":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/role":"pgbouncer"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}},"image":"perconalab/percona-postgresql-operator:main-pgbouncer14","replicas":3}},"users":[{"name":"postgres","password":{"type":"AlphaNumeric"}},{"name":"major-upgrade-14-to-15","password":{"type":"AlphaNumeric"}}]}} + pgv2.percona.com/patroni-version: 4.1.0 + finalizers: + - internal.percona.com/stop-watchers + labels: + e2e: major-upgrade-14-to-15 + managedFields: '[... elided field over 10 lines long ...]' name: major-upgrade-14-to-15 namespace: kuttl-test-safe-skylark +spec: '[... elided field over 10 lines long ...]' status: + conditions: '[... elided field over 10 lines long ...]' + host: major-upgrade-14-to-15-pgbouncer.kuttl-test-safe-skylark.svc + installedCustomExtensions: + - pg_cron + observedGeneration: 8 + patroni: + status: + systemIdentifier: "7609027813903401056" + version: 4.1.0 + patroniVersion: 4.1.0 + pgbackrest: '[... elided field over 10 lines long ...]' pgbouncer: ready: 3 size: 3 postgres: + imageID: docker.io/perconalab/percona-postgresql-operator@sha256:18acf1babe57ab7b0052510984c4f1f5ae8d0c1fafed3fa3ca94cb773129f208 instances: - name: instance1 - ready: 3 + ready: 1 size: 3 - ready: 3 + ready: 1 size: 3 - state: ready + version: 15 + state: initializing case.go:330: resource PerconaPGCluster:kuttl-test-safe-skylark/major-upgrade-14-to-15: .status.postgres.instances.ready: value mismatch, expected: 3 != actual: 1 case.go:330: command "kubectl -n ${NAMESPACE} get pod\\n for pod in $(kubectl -n ${NAMESPA..." exceeded 24 sec timeout, context deadline exceeded logger.go:42: 19:43:08 | major-upgrade-14-to-15 | major-upgrade-14-to-15 events from ns kuttl-test-safe-skylark: logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:11:59 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-65d98588cc from 0 to 1 deployment-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:00 +0000 UTC Normal Pod pg-client-65d98588cc-pgz5b Binding Scheduled Successfully assigned kuttl-test-safe-skylark/pg-client-65d98588cc-pgz5b to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:00 +0000 UTC Normal Pod pg-client-65d98588cc-pgz5b.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:00 +0000 UTC Normal ReplicaSet.apps pg-client-65d98588cc SuccessfulCreate Created pod: pg-client-65d98588cc-pgz5b replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:01 +0000 UTC Normal Pod pg-client-65d98588cc-pgz5b.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 219ms (219ms including waiting). Image size: 564266549 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:01 +0000 UTC Normal Pod pg-client-65d98588cc-pgz5b.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:01 +0000 UTC Normal Pod pg-client-65d98588cc-pgz5b.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:06 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-rvwt-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:06 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-s7px-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:06 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-w64d-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:06 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-w64d-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:06 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-w64d-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-w64d-pgdata" pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:06 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-w64d SuccessfulCreate create Pod major-upgrade-14-to-15-instance1-w64d-0 in StatefulSet major-upgrade-14-to-15-instance1-w64d successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-rvwt-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-rvwt-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-rvwt-pgdata" pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-rvwt SuccessfulCreate create Pod major-upgrade-14-to-15-instance1-rvwt-0 in StatefulSet major-upgrade-14-to-15-instance1-rvwt successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-s7px-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-s7px-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-s7px-pgdata" pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-s7px SuccessfulCreate create Pod major-upgrade-14-to-15-instance1-s7px-0 in StatefulSet major-upgrade-14-to-15-instance1-s7px successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-repo-host SuccessfulCreate create Pod major-upgrade-14-to-15-repo-host-0 in StatefulSet major-upgrade-14-to-15-repo-host successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-repo1 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-safe-skylark/major-upgrade-14-to-15-repo1" pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:07 +0000 UTC Normal PostgresCluster.postgres-operator.crunchydata.com major-upgrade-14-to-15 RepoHostCreated created pgBackRest repository host StatefulSet/major-upgrade-14-to-15-repo-host postgrescluster-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-t36f default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-5b697d6796 SuccessfulCreate Created pod: major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-5b697d6796 SuccessfulCreate Created pod: major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-5b697d6796 SuccessfulCreate Created pod: major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal Deployment.apps major-upgrade-14-to-15-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-14-to-15-pgbouncer-5b697d6796 from 0 to 3 deployment-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:08 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-14-to-15-pgbouncer NoPods No matching pods found controllermanager logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer14" in 269ms (269ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer14" in 217ms (217ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer14" in 301ms (301ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer14" in 254ms (254ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer14" in 202ms (202ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer14" in 176ms (176ms including waiting). Image size: 87586493 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:09 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:10 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-rvwt-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-b36b05fb-242e-41cf-aa88-c9e732f38813 pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:10 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-s7px-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-d4ed36c5-848b-4f6f-b5a7-bbf77cfa0400 pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:10 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-w64d-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:10 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-instance1-w64d-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-4dd21ee4-68e2-4ad2-b8d7-c2f79af6e265 pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:11 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-rvwt-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:11 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-s7px-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-t36f default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:11 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:11 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-fea12dab-de68-4dec-9e96-3531e72e0539 pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:18 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b36b05fb-242e-41cf-aa88-c9e732f38813" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:18 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-d4ed36c5-848b-4f6f-b5a7-bbf77cfa0400" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:18 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-4dd21ee4-68e2-4ad2-b8d7-c2f79af6e265" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:19 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 459ms (459ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 417ms (417ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 250ms (250ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 201ms (201ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 218ms (218ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 150ms (150ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 511ms (511ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 227ms (227ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-14} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 189ms (189ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:22 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-fea12dab-de68-4dec-9e96-3531e72e0539" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 276ms (276ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 229ms (229ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 228ms (228ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest14" in 202ms (202ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:23 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:24 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:24 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest14" in 221ms (221ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:24 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:24 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 289ms (289ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 217ms (217ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 254ms (254ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest14" in 213ms (213ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest14" in 165ms (165ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:25 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 234ms (234ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 277ms (277ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg14-postgres" in 262ms (262ms including waiting). Image size: 543438414 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:27 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:27 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:27 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg14-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:29 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:29 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:33 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-backup-mgrp-d5mhv to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:33 +0000 UTC Normal Job.batch major-upgrade-14-to-15-backup-mgrp SuccessfulCreate Created pod: major-upgrade-14-to-15-backup-mgrp-d5mhv job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:33 +0000 UTC Normal PostgresCluster.postgres-operator.crunchydata.com major-upgrade-14-to-15 StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:34 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:34 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 146ms (147ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:34 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:34 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:35 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv.spec.containers{pgbackrest} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest14" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:35 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv.spec.containers{pgbackrest} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest14" in 186ms (186ms including waiting). Image size: 164315840 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:35 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:35 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-mgrp-d5mhv.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:37 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-14-to-15 UnableToCreateStanzas command terminated with exit code 50: repo1-path = /pgbackrest/repo1 2026-02-20 19:12:37.964 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-02-20 19:12:37.973 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? postgrescluster-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:12:39 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:04 +0000 UTC Normal Job.batch major-upgrade-14-to-15-backup-mgrp Completed Job completed job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:28 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-w64d SuccessfulDelete delete Pod major-upgrade-14-to-15-instance1-w64d-0 in StatefulSet major-upgrade-14-to-15-instance1-w64d successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:29 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-s7px SuccessfulDelete delete Pod major-upgrade-14-to-15-instance1-s7px-0 in StatefulSet major-upgrade-14-to-15-instance1-s7px successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-5b697d6796 SuccessfulDelete Deleted pod: major-upgrade-14-to-15-pgbouncer-5b697d6796-rnc5s replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-5b697d6796 SuccessfulDelete Deleted pod: major-upgrade-14-to-15-pgbouncer-5b697d6796-bwhjd replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-5b697d6796 SuccessfulDelete Deleted pod: major-upgrade-14-to-15-pgbouncer-5b697d6796-vsjcj replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:31 +0000 UTC Normal Deployment.apps major-upgrade-14-to-15-pgbouncer ScalingReplicaSet Scaled down replica set major-upgrade-14-to-15-pgbouncer-5b697d6796 from 3 to 0 deployment-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:34 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-rvwt SuccessfulDelete delete Pod major-upgrade-14-to-15-instance1-rvwt-0 in StatefulSet major-upgrade-14-to-15-instance1-rvwt successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:37 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-14-to-15-set-instance1 NoPods No matching pods found controllermanager logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:44 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:44 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:44 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-repo-host SuccessfulDelete delete Pod major-upgrade-14-to-15-repo-host-0 in StatefulSet major-upgrade-14-to-15-repo-host successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:47 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d Binding Scheduled Successfully assigned kuttl-test-safe-skylark/14-to-15-pgdata-rcn2d to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:47 +0000 UTC Normal Job.batch 14-to-15-pgdata SuccessfulCreate Created pod: 14-to-15-pgdata-rcn2d job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:55 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b36b05fb-242e-41cf-aa88-c9e732f38813" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:56 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:56 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 216ms (216ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:56 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:56 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:57 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-relocator-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:58 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-relocator-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 218ms (218ms including waiting). Image size: 728720234 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:58 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-relocator-14} Created Created container: extension-relocator-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:58 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-relocator-14} Started Started container extension-relocator-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:58 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-installer-14} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:59 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-installer-14} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 224ms (224ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:59 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-installer-14} Created Created container: extension-installer-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:59 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-installer-14} Started Started container extension-installer-14 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:13:59 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-relocator-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:00 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 212ms (212ms including waiting). Image size: 728720234 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:00 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:00 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:00 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:01 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 213ms (213ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:01 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:01 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:02 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:02 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:03 +0000 UTC Normal Pod 14-to-15-pgdata-rcn2d.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:13 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:13 +0000 UTC Normal Job.batch 14-to-15-major-upgrade-14-to-15-instance1-s7px SuccessfulCreate Created pod: 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:13 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:13 +0000 UTC Normal Job.batch 14-to-15-major-upgrade-14-to-15-instance1-w64d SuccessfulCreate Created pod: 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:13 +0000 UTC Normal Job.batch 14-to-15-pgdata Completed Job completed job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:21 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-4dd21ee4-68e2-4ad2-b8d7-c2f79af6e265" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:22 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:22 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:22 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-w64d-mxrx9.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:24 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-d4ed36c5-848b-4f6f-b5a7-bbf77cfa0400" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:25 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:25 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:25 +0000 UTC Normal Pod 14-to-15-major-upgrade-14-to-15-instance1-s7px-gb229.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:25 +0000 UTC Normal Job.batch 14-to-15-major-upgrade-14-to-15-instance1-w64d Completed Job completed job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:28 +0000 UTC Normal Job.batch 14-to-15-major-upgrade-14-to-15-instance1-s7px Completed Job completed job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:34 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-rvwt-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-t36f default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-7798f77544 SuccessfulCreate Created pod: major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69 replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-7798f77544 SuccessfulCreate Created pod: major-upgrade-14-to-15-pgbouncer-7798f77544-tn878 replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal ReplicaSet.apps major-upgrade-14-to-15-pgbouncer-7798f77544 SuccessfulCreate Created pod: major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8 replicaset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal Deployment.apps major-upgrade-14-to-15-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-14-to-15-pgbouncer-7798f77544 from 0 to 3 deployment-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:35 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" in 178ms (178ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" in 168ms (168ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-hlmv8.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" in 224ms (224ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" in 238ms (238ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-mfr69.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" in 217ms (217ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer15" in 203ms (203ms including waiting). Image size: 87597676 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:36 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbouncer-7798f77544-tn878.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:42 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b36b05fb-242e-41cf-aa88-c9e732f38813" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:43 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:43 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 215ms (215ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:43 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:43 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:44 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:44 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 184ms (184ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:44 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:44 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:45 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:45 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 209ms (209ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:45 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:45 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:45 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-fea12dab-de68-4dec-9e96-3531e72e0539" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:46 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:46 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 242ms (242ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:46 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:46 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:46 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:47 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:47 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 235ms (235ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:47 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:47 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:47 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 220ms (220ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:47 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:47 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 214ms (214ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 195ms (195ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:48 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:49 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:49 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 198ms (198ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:49 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:49 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:49 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:49 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 212ms (212ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:49 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:49 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:54 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-s7px-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:54 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-w64d-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-t36f default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:55 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-backup-7c6q-k2tnd to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:55 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:55 +0000 UTC Normal Job.batch major-upgrade-14-to-15-backup-7c6q SuccessfulCreate Created pod: major-upgrade-14-to-15-backup-7c6q-k2tnd job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:56 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 199ms (199ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:56 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:56 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:57 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:57 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 208ms (208ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:57 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:14:57 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-7c6q-k2tnd.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:02 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-d4ed36c5-848b-4f6f-b5a7-bbf77cfa0400" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:03 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:03 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 187ms (187ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:03 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:03 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 207ms (207ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:06 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:06 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 221ms (221ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:06 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:06 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:07 +0000 UTC Normal Job.batch major-upgrade-14-to-15-backup-7c6q Completed Job completed job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:07 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:07 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 185ms (185ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:07 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:07 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:09 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:09 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 242ms (242ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:09 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:09 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:10 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:10 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 220ms (220ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:10 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:10 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:10 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:13 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:14 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-4dd21ee4-68e2-4ad2-b8d7-c2f79af6e265" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:15 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:16 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 257ms (257ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:16 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:16 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:17 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:17 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 233ms (233ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:17 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:17 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:18 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:18 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 180ms (180ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:18 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:18 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:19 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:19 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 197ms (197ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:19 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:19 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 211ms (211ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 202ms (202ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:25 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:43 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:43 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:43 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-repo2 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:43 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-14-to-15 UnableToCreateStanzas command terminated with exit code 1: postgres operator error: pgBackRest stale volume-backed repo configuration postgrescluster-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:45 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-repo2 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:45 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-repo2 Provisioning External provisioner is provisioning volume for claim "kuttl-test-safe-skylark/major-upgrade-14-to-15-repo2" pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:49 +0000 UTC Normal PersistentVolumeClaim major-upgrade-14-to-15-repo2 ProvisioningSucceeded Successfully provisioned volume pvc-a5e3511e-f6b3-4369-8788-89a65c13583b pd.csi.storage.gke.io_gke-be0d46e4d03845f1a28b-8421-f73f-vm_4d27c551-aea6-4acf-a7c4-b95f752ddcae logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:50 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-repo-host-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:57 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-a5e3511e-f6b3-4369-8788-89a65c13583b" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:58 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:58 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 219ms (219ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:58 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:59 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:59 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:59 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 219ms (219ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:59 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:15:59 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:00 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:00 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 208ms (208ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:00 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:00 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:00 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:00 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 192ms (192ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:00 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:01 +0000 UTC Normal Pod major-upgrade-14-to-15-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:54 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com major-upgrade-14-to-15 UnableToCreateStanzas command terminated with exit code 55: repo2-path = /pgbackrest/repo2 2026-02-20 19:16:53.972 P00 ERROR: [049]: timeout connecting to 'major-upgrade-14-to-15-repo-host-0.major-upgrade-14-to-15-pods.kuttl-test-safe-skylark.svc.cluster.local:8432 (10.50.194.57)' 2026-02-20 19:16:54.005 P00 ERROR: [055]: unable to load info file '/pgbackrest/repo2/archive/db/archive.info' or '/pgbackrest/repo2/archive/db/archive.info.copy': FileMissingError: raised from remote-0 tls protocol on 'major-upgrade-14-to-15-repo-host-0.major-upgrade-14-to-15-pods.kuttl-test-safe-skylark.svc.cluster.local': unable to open missing file '/pgbackrest/repo2/archive/db/archive.info' for read FileMissingError: raised from remote-0 tls protocol on 'major-upgrade-14-to-15-repo-host-0.major-upgrade-14-to-15-pods.kuttl-test-safe-skylark.svc.cluster.local': unable to open missing file '/pgbackrest/repo2/archive/db/archive.info.copy' for read HINT: archive.info cannot be opened but is required to push/get WAL segments. HINT: is archive_command configured correctly in postgresql.conf? HINT: has a stanza-create been performed? HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme. postgrescluster-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:56 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-backup-d4s9-sjqsq to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:56 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:56 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 244ms (244ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:56 +0000 UTC Normal Job.batch major-upgrade-14-to-15-backup-d4s9 SuccessfulCreate Created pod: major-upgrade-14-to-15-backup-d4s9-sjqsq job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:57 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:57 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:57 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:58 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 195ms (195ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:58 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:16:58 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-d4s9-sjqsq.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:25 +0000 UTC Normal Job.batch major-upgrade-14-to-15-backup-d4s9 Completed Job completed job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:32 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-backup-vs9w-rrwh5 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:32 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:32 +0000 UTC Normal Job.batch major-upgrade-14-to-15-backup-vs9w SuccessfulCreate Created pod: major-upgrade-14-to-15-backup-vs9w-rrwh5 job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:33 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 242ms (242ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:33 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:33 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:33 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:34 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 200ms (200ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:34 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:34 +0000 UTC Normal Pod major-upgrade-14-to-15-backup-vs9w-rrwh5.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:17:43 +0000 UTC Normal Job.batch major-upgrade-14-to-15-backup-vs9w Completed Job completed job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:10 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-pgbackrest-restore-r6hpr to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-t36f default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:10 +0000 UTC Normal Job.batch major-upgrade-14-to-15-pgbackrest-restore SuccessfulCreate Created pod: major-upgrade-14-to-15-pgbackrest-restore-r6hpr job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:11 +0000 UTC Warning Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr FailedAttachVolume Multi-Attach error for volume "pvc-b36b05fb-242e-41cf-aa88-c9e732f38813" Volume is already exclusively attached to one node and can't be attached to another attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:26 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b36b05fb-242e-41cf-aa88-c9e732f38813" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:28 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:28 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest15" in 261ms (261ms including waiting). Image size: 164316601 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:28 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:28 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:28 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr.spec.containers{pgbackrest-restore} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:28 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr.spec.containers{pgbackrest-restore} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 258ms (258ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:29 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr.spec.containers{pgbackrest-restore} Created Created container: pgbackrest-restore kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:29 +0000 UTC Normal Pod major-upgrade-14-to-15-pgbackrest-restore-r6hpr.spec.containers{pgbackrest-restore} Started Started container pgbackrest-restore kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:37 +0000 UTC Normal Job.batch major-upgrade-14-to-15-pgbackrest-restore Completed Job completed job-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:38 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-rvwt-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-5t5d default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:38 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-rvwt-0 FailedAttachVolume Multi-Attach error for volume "pvc-b36b05fb-242e-41cf-aa88-c9e732f38813" Volume is already exclusively attached to one node and can't be attached to another attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:38 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-rvwt SuccessfulCreate create Pod major-upgrade-14-to-15-instance1-rvwt-0 in StatefulSet major-upgrade-14-to-15-instance1-rvwt successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:58 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b36b05fb-242e-41cf-aa88-c9e732f38813" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:59 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:59 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 230ms (230ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:59 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:18:59 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:01 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:01 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 232ms (232ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:01 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:01 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:02 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:02 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 214ms (214ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:02 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:02 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:03 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:03 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 186ms (186ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:03 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:03 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:04 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:04 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 213ms (213ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:04 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:04 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 211ms (211ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:05 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:08 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-rvwt-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:11 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-s7px-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-g005 default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:11 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-s7px SuccessfulCreate create Pod major-upgrade-14-to-15-instance1-s7px-0 in StatefulSet major-upgrade-14-to-15-instance1-s7px successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:11 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0 Binding Scheduled Successfully assigned kuttl-test-safe-skylark/major-upgrade-14-to-15-instance1-w64d-0 to gke-jen-pg-1454-bcdf84ed-default-pool-66e0efff-t36f default-scheduler logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:11 +0000 UTC Normal StatefulSet.apps major-upgrade-14-to-15-instance1-w64d SuccessfulCreate create Pod major-upgrade-14-to-15-instance1-w64d-0 in StatefulSet major-upgrade-14-to-15-instance1-w64d successful statefulset-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:19 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-d4ed36c5-848b-4f6f-b5a7-bbf77cfa0400" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:19 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-4dd21ee4-68e2-4ad2-b8d7-c2f79af6e265" attachdetach-controller logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 187ms (187ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 195ms (195ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:20 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:21 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 207ms (207ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 284ms (284ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:22 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 218ms (218ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 164ms (164ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-15} Created Created container: extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-relocator-15} Started Started container extension-relocator-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:23 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-15} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:24 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 160ms (160ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:24 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:24 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:24 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:24 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-15} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1454-bcdf84ed0" in 226ms (226ms including waiting). Image size: 88552410 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:24 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-15} Created Created container: extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:24 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{extension-installer-15} Started Started container extension-installer-15 kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:24 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 231ms (231ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 218ms (218ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:25 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 229ms (229ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" in 204ms (204ms including waiting). Image size: 543980232 bytes. kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Started Started container database kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:26 +0000 UTC Normal Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg15-postgres" kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:30 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-s7px-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | 2026-02-20 19:19:30 +0000 UTC Warning Pod major-upgrade-14-to-15-instance1-w64d-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 19:43:08 | major-upgrade-14-to-15 | Deleting namespace "kuttl-test-safe-skylark" === NAME kuttl harness.go:404: run tests finished harness.go:511: cleaning up harness.go:568: removing temp folder: "" --- FAIL: kuttl (1935.93s) --- FAIL: kuttl/harness (0.00s) --- FAIL: kuttl/harness/major-upgrade-14-to-15 (1935.10s) FAIL