=== RUN kuttl harness.go:460: starting setup harness.go:258: running tests using configured kubeconfig. harness.go:281: Successful connection to cluster at: https://35.225.159.147 harness.go:366: running tests harness.go:77: going to run test suite with timeout of 600 seconds for each step harness.go:378: testsuite: e2e-tests/tests has 58 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-logical-replica === PAUSE kuttl/harness/major-upgrade-logical-replica === CONT kuttl/harness/major-upgrade-logical-replica logger.go:42: 11:53:17 | major-upgrade-logical-replica | Ignoring "conf": does not begin with a number followed by a dash. logger.go:42: 11:53:17 | major-upgrade-logical-replica | Creating namespace "kuttl-test-verified-possum" logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | + source ../../functions logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ realpath ../../.. logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++ BUSYBOX_VER=1.36 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ pwd logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++ test_name=major-upgrade-logical-replica logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/vars.sh logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export GIT_BRANCH=PR-1814 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ GIT_BRANCH=PR-1814 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export VERSION=PR-1814-d67db8e10 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ VERSION=PR-1814-d67db8e10 logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -z '' ]] logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ detect_k8s_platform logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ local detected_platform logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ local xtrace_enabled=false logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ [[ -o xtrace ]] logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ xtrace_enabled=true logger.go:42: 11:53:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ set +o xtrace logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ PLATFORM=gke logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export PLATFORM logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_VER=18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_DISTRIBUTION= logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_DISTRIBUTION= logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ '' == community ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export CPGO_VERSION=5.8.7 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ CPGO_VERSION=5.8.7 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export VAULT_VER=0.32.0 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ VAULT_VER=0.32.0 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ printenv logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gdate logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ which date logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gsed logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | ++++ which sed logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + init_temp_dir logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + deploy_operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + local cw_prefix= logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + destroy_operator logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | Error from server (NotFound): deployments.apps "percona-postgresql-operator" not found logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + true logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 11:53:20 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 11:53:21 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 11:53:21 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 11:53:21 | major-upgrade-logical-replica/0-deploy-operator | + create_namespace pg-operator logger.go:42: 11:53:21 | major-upgrade-logical-replica/0-deploy-operator | + local namespace=pg-operator logger.go:42: 11:53:21 | major-upgrade-logical-replica/0-deploy-operator | + [[ gke == openshift ]] logger.go:42: 11:53:21 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 11:53:22 | major-upgrade-logical-replica/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 11:53:22 | major-upgrade-logical-replica/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 11:53:23 | major-upgrade-logical-replica/0-deploy-operator | namespace/pg-operator created logger.go:42: 11:53:23 | major-upgrade-logical-replica/0-deploy-operator | + cw_prefix=cw- logger.go:42: 11:53:23 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy/crd.yaml logger.go:42: 11:53:24 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 11:53:24 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 11:53:27 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 11:53:27 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 11:53:27 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 11:53:28 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.upstream.pgv2.percona.com serverside-applied logger.go:42: 11:53:28 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.upstream.pgv2.percona.com serverside-applied logger.go:42: 11:53:30 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 11:53:30 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy/cw-rbac.yaml logger.go:42: 11:53:31 | major-upgrade-logical-replica/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | + local disable_telemetry=true logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | + '[' major-upgrade-logical-replica == telemetry-transfer ']' logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy/cw-operator.yaml logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 11:53:32 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 11:53:33 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 11:53:33 | major-upgrade-logical-replica/0-deploy-operator | + deploy_client logger.go:42: 11:53:33 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n kuttl-test-verified-possum apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf/client.yaml logger.go:42: 11:53:35 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 11:53:35 | major-upgrade-logical-replica/0-deploy-operator | + deploy_s3_secrets logger.go:42: 11:53:35 | major-upgrade-logical-replica/0-deploy-operator | + set +o xtrace logger.go:42: 11:53:35 | major-upgrade-logical-replica/0-deploy-operator | secret/major-upgrade-logical-replica-pgbackrest-secrets created logger.go:42: 11:53:36 | major-upgrade-logical-replica/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 11:53:37 | major-upgrade-logical-replica/0-deploy-operator | secret/do-spaces-secret created logger.go:42: 11:53:37 | major-upgrade-logical-replica/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 11:53:38 | major-upgrade-logical-replica/0-deploy-operator | secret/azure-secret created logger.go:42: 11:53:39 | major-upgrade-logical-replica/0-deploy-operator | secret/minio-secret created logger.go:42: 11:53:40 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 11:53:40 | major-upgrade-logical-replica/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 11:53:40 | major-upgrade-logical-replica/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 11:53:40 | major-upgrade-logical-replica/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 11:53:40 | major-upgrade-logical-replica/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 11:53:40 | major-upgrade-logical-replica/0-deploy-operator | ASSERT PASS logger.go:42: 11:53:40 | major-upgrade-logical-replica/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | starting test step 1-create-cluster logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions postgres_image=$(get_container_image "postgres" 17) pgbouncer_image=$(get_container_image "pgbouncer" 17) pgbackrest_image=$(get_container_image "pgbackrest" 17) get_cr \ | yq eval " .spec.postgresVersion = 17 | .spec.image = \"${postgres_image}\" | .spec.proxy.pgBouncer.image = \"${pgbouncer_image}\" | .spec.backups.pgbackrest.image = \"${pgbackrest_image}\" | .spec.patroni.removeDataDirectoryOnDivergedTimelines = true" \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | + source ../../functions logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ realpath ../../.. logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++ BUSYBOX_VER=1.36 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++++ pwd logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++ test_name=major-upgrade-logical-replica logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/vars.sh logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export GIT_BRANCH=PR-1814 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ GIT_BRANCH=PR-1814 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export VERSION=PR-1814-d67db8e10 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ VERSION=PR-1814-d67db8e10 logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -z '' ]] logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++++ detect_k8s_platform logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++++ local detected_platform logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++++ local xtrace_enabled=false logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++++ [[ -o xtrace ]] logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++++ xtrace_enabled=true logger.go:42: 11:53:40 | major-upgrade-logical-replica/1-create-cluster | ++++ set +o xtrace logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ PLATFORM=gke logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export PLATFORM logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_VER=18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ PG_VER=18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_DISTRIBUTION= logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ PG_DISTRIBUTION= logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ '' == community ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export CPGO_VERSION=5.8.7 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ CPGO_VERSION=5.8.7 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export VAULT_VER=0.32.0 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ VAULT_VER=0.32.0 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ printenv logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ which gdate logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ which date logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ which gsed logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++++ which sed logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image postgres 17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local component=postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1814-d67db8e10 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbouncer ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbackrest ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == upgrade ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == postgis ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + postgres_image=docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbouncer 17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbouncer logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1814-d67db8e10 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbouncer == pgbouncer ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + pgbouncer_image=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbackrest 17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbackrest logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/percona/* ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1814-d67db8e10 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1814-d67db8e10 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbouncer ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbackrest ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + pgbackrest_image=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + get_cr logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + local cr_name=major-upgrade-logical-replica logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + local repo_path= logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + local source_path= logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + local platform=gke logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + local cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + local platform_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + yq eval $'\n .spec.postgresVersion = 17 |\n .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" |\n .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" |\n .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" |\n .spec.patroni.removeDataDirectoryOnDivergedTimelines = true' logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + local spaces_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-spaces.yaml logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + kubectl -n kuttl-test-verified-possum apply -f - logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == doks ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + crs=('/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy/cr.yaml') logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + local crs logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + crs+=($cr_file) logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + yq eval-all $'\n\t\tselect(fileIndex == 0) * (select(fileIndex == 1) // {}) |\n\t\t.metadata.name = "major-upgrade-logical-replica" |\n\t\t.metadata.labels = {"e2e":"major-upgrade-logical-replica"} |\n\t\t.spec.postgresVersion = 18 |\n\t\t.spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.users += [{"name":"major-upgrade-logical-replica","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" |\n\t\t.spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" |\n\t\t.spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" |\n\t\t.spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" |\n\t\t.spec.logcollector.image = "docker.io/perconalab/fluentbit:main-logcollector" |\n\t\t.spec.pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest" |\n\t\t.spec.pmm.secret = "major-upgrade-logical-replica-pmm-secret" |\n\t\t.spec.pmm.customClusterName = "major-upgrade-logical-replica-pmm-custom-name" |\n\t\t.spec.pmm.postgresParams = "--environment=dev-postgres"\n\t' /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy/cr.yaml /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||pg-operator-testing|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == openshift ]] logger.go:42: 11:53:42 | major-upgrade-logical-replica/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 11:53:44 | major-upgrade-logical-replica/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica created logger.go:42: 11:54:45 | major-upgrade-logical-replica/1-create-cluster | test step completed 1-create-cluster logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | starting test step 2-write-data logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:${PASS}@${HOST}" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:${PASS}@${HOST}"] logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | + source ../../functions logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ realpath ../../.. logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++ BUSYBOX_VER=1.36 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++++ pwd logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++ test_name=major-upgrade-logical-replica logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/vars.sh logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export GIT_BRANCH=PR-1814 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ GIT_BRANCH=PR-1814 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export VERSION=PR-1814-d67db8e10 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ VERSION=PR-1814-d67db8e10 logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | +++ [[ -z '' ]] logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++++ detect_k8s_platform logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++++ local detected_platform logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++++ local xtrace_enabled=false logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++++ [[ -o xtrace ]] logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++++ xtrace_enabled=true logger.go:42: 11:54:45 | major-upgrade-logical-replica/2-write-data | ++++ set +o xtrace logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ PLATFORM=gke logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export PLATFORM logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ -n '' ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export PG_VER=18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ PG_VER=18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export PG_DISTRIBUTION= logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ PG_DISTRIBUTION= logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ '' == community ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_VER=14 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export CPGO_VERSION=5.8.7 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ CPGO_VERSION=5.8.7 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export VAULT_VER=0.32.0 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ VAULT_VER=0.32.0 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ printenv logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ which gdate logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ which date logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ date=/usr/sbin/date logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ which gsed logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++++ which sed logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 11:54:47 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-verified-possum get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | + PASS=cIrIq0O4eX3gcxZCnXoUDSXg logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-verified-possum get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | + HOST=major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 11:54:48 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-verified-possum get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 11:54:49 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-verified-possum exec pg-client-d7c7bb45b-gljfp -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc'\''' logger.go:42: 11:54:52 | major-upgrade-logical-replica/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 11:54:52 | major-upgrade-logical-replica/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 11:54:52 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 11:54:52 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 11:54:52 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 11:54:52 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-verified-possum get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 11:54:52 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-verified-possum exec pg-client-d7c7bb45b-gljfp -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc'\''' logger.go:42: 11:54:53 | major-upgrade-logical-replica/2-write-data | test step completed 2-write-data logger.go:42: 11:54:53 | major-upgrade-logical-replica/3-create-seed-backup | starting test step 3-create-seed-backup logger.go:42: 11:54:53 | major-upgrade-logical-replica/3-create-seed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # The pgbackrest bootstrap Job restores the latest full backup, and the # one taken at cluster creation predates the rows written above. Take a # new one so the seeded data is actually in the backup being restored. cat </dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:32 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:34 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:36 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:40 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:43 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:45 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:47 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:49 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:51 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:52 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:58:53 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:58:55 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:58:56 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:58:57 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:58:59 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:00 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:01 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:03 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:04 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:05 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:07 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:08 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:08 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:11 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:12 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:15 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:15 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:16 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:19 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:19 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:20 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:23 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:23 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:24 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:27 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:27 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:28 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:30 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:31 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:32 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:34 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:35 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:36 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-4bt5b" is waiting to start: PodInitializing logger.go:42: 11:59:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:39 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:40 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:43 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 11:59:44 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 11:59:46 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:47 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 11:59:48 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 11:59:50 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:51 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:52 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 11:59:54 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:55 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 11:59:56 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 11:59:58 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 11:59:59 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:00 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:02 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:03 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:04 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:07 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:07 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:08 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:11 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:13 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:15 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:16 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:17 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:19 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:20 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:21 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:23 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:24 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:25 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:27 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:28 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:29 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:32 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:33 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:34 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:36 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:37 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:38 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:40 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:41 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:42 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:44 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:45 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:46 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:49 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:49 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:50 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:53 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:53 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:55 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:00:57 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:00:58 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:00:59 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:01:01 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:01:02 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:03 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:01:05 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:01:06 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:07 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:01:09 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:01:10 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:11 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:01:13 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:01:14 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:15 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:01:17 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:01:18 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:19 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:01:21 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:01:22 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:23 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:01:25 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 12:01:26 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | real 0m1.725s logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | user 0m0.073s logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | sys 0m0.114s logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | real 0m9.594s logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | sys 0m0.459s logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 12:01:27 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 12:01:28 | major-upgrade-logical-replica/5-upgrade | test step completed 5-upgrade logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | starting test step 6-verify-replica-invalidated logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-logical-replica primary name) kubectl -n "${NAMESPACE}" exec "${primary}" -- /pgdata/delete_old_cluster.sh # pg_upgrade only rewrites the data directories of the instance sets, so the # replica still holds a PostgreSQL 17 one. It has to be reported as broken # rather than handed the new binary and left to crash. retry 60 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") reason=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].reason}") echo "logical replica analytics: ${state}/${reason}" [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]] ' message=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].message}') echo "message: ${message}" echo "${message}" | grep -q "remove it from spec.logicalReplicas" # The StatefulSet is stopped, so nothing is left crash-looping against a data # directory the new binary cannot open. retry 30 5 bash -c ' replicas=$(kubectl -n "'"${NAMESPACE}"'" get statefulset \ major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}") echo "logical replica statefulset replicas: ${replicas}" [[ ${replicas} == "0" ]] ' # pg_upgrade carries the logical slots over to the new cluster. Nothing will # ever read from them again, so they must not be left pinning WAL. PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) no_slots_left() { local got got=$(run_psql \ "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ "postgres://postgres:${PASS}@${HOST}" | xargs) echo "logical slots left on the primary: ${got}" [[ ${got} == "0" ]] } retry 30 5 no_slots_left] logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | + source ../../functions logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ realpath ../../.. logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ BUSYBOX_VER=1.36 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ pwd logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ test_name=major-upgrade-logical-replica logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/vars.sh logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export GIT_BRANCH=PR-1814 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ GIT_BRANCH=PR-1814 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VERSION=PR-1814-d67db8e10 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VERSION=PR-1814-d67db8e10 logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ SKIP_TEST_WARNINGS=true logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -z '' ]] logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ detect_k8s_platform logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local detected_platform logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local xtrace_enabled=false logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ [[ -o xtrace ]] logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ xtrace_enabled=true logger.go:42: 12:01:28 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ set +o xtrace logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PLATFORM=gke logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PLATFORM logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ ! -n '' ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -n '' ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_VER=18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_VER=18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_DISTRIBUTION= logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_DISTRIBUTION= logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ '' == community ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export BUCKET=pg-operator-testing logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ BUCKET=pg-operator-testing logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_TAG=1.4.0 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_TAG=1.4.0 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_VER=14 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_VER=14 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export CPGO_VERSION=5.8.7 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ CPGO_VERSION=5.8.7 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export MINIO_VER=5.4.0 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ MINIO_VER=5.4.0 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VAULT_VER=0.32.0 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VAULT_VER=0.32.0 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME=docker.io logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME=docker.io logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ printenv logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ grep -E '^IMAGE' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ awk -F= '{print $1}' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gdate logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which date logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ date=/usr/sbin/date logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gsed logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which sed logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ sed=/usr/sbin/sed logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_pod_by_role major-upgrade-logical-replica primary name logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local cluster=major-upgrade-logical-replica logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local role=primary logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local parameter=name logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ case ${parameter} in logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'jsonpath={.items[].metadata.name}' logger.go:42: 12:01:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl get pods --namespace kuttl-test-verified-possum --selector=postgres-operator.crunchydata.com/role=primary,postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica -o 'jsonpath={.items[].metadata.name}' logger.go:42: 12:01:31 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ echo major-upgrade-logical-replica-instance1-wjxz-0 logger.go:42: 12:01:31 | major-upgrade-logical-replica/6-verify-replica-invalidated | + primary=major-upgrade-logical-replica-instance1-wjxz-0 logger.go:42: 12:01:31 | major-upgrade-logical-replica/6-verify-replica-invalidated | + kubectl -n kuttl-test-verified-possum exec major-upgrade-logical-replica-instance1-wjxz-0 -- /pgdata/delete_old_cluster.sh logger.go:42: 12:01:33 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 60 10 bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 12:01:33 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=60 logger.go:42: 12:01:33 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=10 logger.go:42: 12:01:33 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 12:01:33 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 12:01:33 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 12:01:34 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica analytics: broken/SourceUpgraded logger.go:42: 12:01:34 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-verified-possum get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].message}' logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + message='this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + grep -q 'remove it from spec.logicalReplicas' logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 bash -c $'\n replicas=$(kubectl -n "kuttl-test-verified-possum" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n replicas=$(kubectl -n "kuttl-test-verified-possum" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica statefulset replicas: 0 logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 12:01:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-verified-possum get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + PASS=cIrIq0O4eX3gcxZCnXoUDSXg logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-verified-possum get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + HOST=major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 no_slots_left logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + no_slots_left logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local got logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ run_psql 'SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'command=SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ xargs logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local uri=postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local password= logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ get_client_pod logger.go:42: 12:01:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl -n kuttl-test-verified-possum get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 12:01:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-verified-possum exec pg-client-d7c7bb45b-gljfp -- bash -c 'printf '\''SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc' logger.go:42: 12:01:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | + got=0 logger.go:42: 12:01:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'logical slots left on the primary: 0' logger.go:42: 12:01:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical slots left on the primary: 0 logger.go:42: 12:01:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | + [[ 0 == 0 ]] logger.go:42: 12:01:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | test step completed 6-verify-replica-invalidated logger.go:42: 12:01:38 | major-upgrade-logical-replica/7-create-reseed-backup | starting test step 7-create-reseed-backup logger.go:42: 12:01:38 | major-upgrade-logical-replica/7-create-reseed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # pg_upgrade left the repository holding only pre-upgrade backups, so the # bootstrap Job of the reseed below would restore a PostgreSQL 17 data # directory. Take a full backup of the upgraded cluster first. cat </dev/null 2>&1 && return 1 kubectl -n "${NAMESPACE}" get pvc major-upgrade-logical-replica-lr-analytics-pgdata >/dev/null 2>&1 && return 1 return 0 } retry 60 5 status_gone retry 60 5 objects_gone kubectl -n "${NAMESPACE}" patch pg major-upgrade-logical-replica --type=merge --patch '{ "spec": { "logicalReplicas": [ { "name": "analytics", "databases": ["myapp"], "bootstrapMethod": "pgbackrest", "dataVolumeClaimSpec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}} } } ] } }'] logger.go:42: 12:02:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 12:02:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/vars.sh logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1814 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1814 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1814-d67db8e10 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1814-d67db8e10 logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 12:02:17 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 12:02:19 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-verified-possum patch pg major-upgrade-logical-replica --type=json --patch '[{"op": "remove", "path": "/spec/logicalReplicas"}]' logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 status_gone logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-verified-possum get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + got='[{"invalidatedAt":"2026-09-03T12:00:40Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T11:57:45Z","state":"broken"}]' logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\''[{"invalidatedAt":"2026-09-03T12:00:40Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T11:57:45Z","state":"broken"}]'\''' logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '[{"invalidatedAt":"2026-09-03T12:00:40Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T11:57:45Z","state":"broken"}]' logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z [{"invalidatedAt":"2026-09-03T12:00:40Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T11:57:45Z","state":"broken"}] ]] logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ [{"invalidatedAt":"2026-09-03T12:00:40Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T11:57:45Z","state":"broken"}] == \[\] ]] logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 60 ]] logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 12:02:20 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 5 logger.go:42: 12:02:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 12:02:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 12:02:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-verified-possum get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + got= logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\'''\''' logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '' logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z '' ]] logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 objects_gone logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + objects_gone logger.go:42: 12:02:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-verified-possum get statefulset major-upgrade-logical-replica-lr-analytics logger.go:42: 12:02:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-verified-possum get pvc major-upgrade-logical-replica-lr-analytics-pgdata logger.go:42: 12:02:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + return 0 logger.go:42: 12:02:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-verified-possum patch pg major-upgrade-logical-replica --type=merge --patch $'{\n "spec": {\n "logicalReplicas": [\n {\n "name": "analytics",\n "databases": ["myapp"],\n "bootstrapMethod": "pgbackrest",\n "dataVolumeClaimSpec": {\n "accessModes": ["ReadWriteOnce"],\n "resources": {"requests": {"storage": "1Gi"}}\n }\n }\n ]\n }\n}' logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | running command: [sh -c set -o errexit set -o xtrace source ../../functions retry 90 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") echo "logical replica analytics state: ${state}" [[ ${state} == "ready" ]] ' # Seeded from the upgraded cluster, so it is stamped with the new version. seeded=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}') echo "seeded at PostgreSQL: ${seeded}" [[ ${seeded} == "18" ]] kubectl -n "${NAMESPACE}" wait --for=condition=Ready pod \ -l "pgv2.percona.com/logical-replica=analytics" --timeout=300s PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) REPLICA="major-upgrade-logical-replica-lr-analytics.${NAMESPACE}.svc" # Replication actually streams again on the upgraded cluster. run_psql \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' \ "postgres://postgres:${PASS}@${HOST}" row_streamed() { local got got=$(run_psql \ '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' \ "postgres://postgres:${PASS}@${REPLICA}" | xargs) echo "streamed rows on the replica: ${got}" [[ ${got} == "1" ]] } retry 30 5 row_streamed] logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/vars.sh logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1814 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1814 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1814-d67db8e10 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1814-d67db8e10 logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 12:02:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 90 10 bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=90 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=10 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 12:02:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 12:02:31 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 12:02:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 90 ]] logger.go:42: 12:02:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 12:02:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 12:02:41 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 12:02:42 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 12:02:42 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 2 -ge 90 ]] logger.go:42: 12:02:42 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 12:02:42 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 12:02:52 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 12:02:52 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 12:02:52 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 3 -ge 90 ]] logger.go:42: 12:02:52 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 12:02:52 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 12:03:02 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 12:03:03 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 12:03:03 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 4 -ge 90 ]] logger.go:42: 12:03:03 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 12:03:03 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 12:03:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 12:03:13 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 12:03:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 5 -ge 90 ]] logger.go:42: 12:03:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 12:03:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 12:03:23 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-verified-possum" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 12:03:24 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: ready logger.go:42: 12:03:24 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-verified-possum get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}' logger.go:42: 12:03:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + seeded=18 logger.go:42: 12:03:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'seeded at PostgreSQL: 18' logger.go:42: 12:03:25 | major-upgrade-logical-replica/8-reseed-logical-replica | seeded at PostgreSQL: 18 logger.go:42: 12:03:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 18 == 18 ]] logger.go:42: 12:03:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-verified-possum wait --for=condition=Ready pod -l pgv2.percona.com/logical-replica=analytics --timeout=300s logger.go:42: 12:03:26 | major-upgrade-logical-replica/8-reseed-logical-replica | pod/major-upgrade-logical-replica-lr-analytics-0 condition met logger.go:42: 12:03:26 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 12:03:26 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 12:03:26 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-verified-possum get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 12:03:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + PASS=cIrIq0O4eX3gcxZCnXoUDSXg logger.go:42: 12:03:26 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 12:03:26 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 12:03:26 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-verified-possum get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + HOST=major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + REPLICA=major-upgrade-logical-replica-lr-analytics.kuttl-test-verified-possum.svc logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + local uri=postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + local password= logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_client_pod logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-verified-possum get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 12:03:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-verified-possum exec pg-client-d7c7bb45b-gljfp -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-primary.kuttl-test-verified-possum.svc' logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 30 5 row_streamed logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=30 logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + row_streamed logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ run_psql '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-lr-analytics.kuttl-test-verified-possum.svc logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local 'command=\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ xargs logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local uri=postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-lr-analytics.kuttl-test-verified-possum.svc logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local password= logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ get_client_pod logger.go:42: 12:03:28 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ kubectl -n kuttl-test-verified-possum get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 12:03:29 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-verified-possum exec pg-client-d7c7bb45b-gljfp -- bash -c 'printf '\''\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:cIrIq0O4eX3gcxZCnXoUDSXg@major-upgrade-logical-replica-lr-analytics.kuttl-test-verified-possum.svc' logger.go:42: 12:03:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + got=1 logger.go:42: 12:03:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'streamed rows on the replica: 1' logger.go:42: 12:03:30 | major-upgrade-logical-replica/8-reseed-logical-replica | streamed rows on the replica: 1 logger.go:42: 12:03:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 == 1 ]] logger.go:42: 12:03:30 | major-upgrade-logical-replica/8-reseed-logical-replica | test step completed 8-reseed-logical-replica logger.go:42: 12:03:30 | major-upgrade-logical-replica/99-remove-cluster-gracefully | starting test step 99-remove-cluster-gracefully logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | running command: [sh -c set -o errexit set -o xtrace source ../../functions remove_all_finalizers check_operator_panic destroy_operator] logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + source ../../functions logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ realpath ../../.. logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ BUSYBOX_VER=1.36 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ pwd logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ test_name=major-upgrade-logical-replica logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/vars.sh logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/deploy logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/conf logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export GIT_BRANCH=PR-1814 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ GIT_BRANCH=PR-1814 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VERSION=PR-1814-d67db8e10 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VERSION=PR-1814-d67db8e10 logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ SKIP_TEST_WARNINGS=true logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -z '' ]] logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ detect_k8s_platform logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local detected_platform logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local xtrace_enabled=false logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ [[ -o xtrace ]] logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ xtrace_enabled=true logger.go:42: 12:03:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ set +o xtrace logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PLATFORM=gke logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PLATFORM logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ ! -n '' ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -n '' ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_VER=18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_VER=18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_DISTRIBUTION= logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_DISTRIBUTION= logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ '' == community ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export BUCKET=pg-operator-testing logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ BUCKET=pg-operator-testing logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_TAG=1.4.0 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_TAG=1.4.0 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_VER=14 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_VER=14 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export CPGO_VERSION=5.8.7 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ CPGO_VERSION=5.8.7 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export MINIO_VER=5.4.0 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ MINIO_VER=5.4.0 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VAULT_VER=0.32.0 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VAULT_VER=0.32.0 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME=docker.io logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME=docker.io logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ printenv logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ grep -E '^IMAGE' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ awk -F= '{print $1}' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1814-d67db8e10 == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10 logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gdate logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which date logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ date=/usr/sbin/date logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gsed logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1814/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which sed logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ sed=/usr/sbin/sed logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + remove_all_finalizers logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + resource_types=("pg-restore" "pg-backup" "pg") logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-restore resources' logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-restore resources logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-verified-possum get pg-restore -o json logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 12:03:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 12:03:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 12:03:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-backup resources' logger.go:42: 12:03:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-backup resources logger.go:42: 12:03:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-verified-possum get pg-backup -o json logger.go:42: 12:03:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 12:03:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 12:03:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 12:03:43 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 12:03:43 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-verified-possum get pg-backup major-upgrade-logical-replica-backup-dhk7-48bsh -o yaml logger.go:42: 12:03:43 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 12:03:43 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 12:03:43 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 12:03:43 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 12:03:43 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-verified-possum get pg-backup major-upgrade-logical-replica-backup-k7nc-r6j5f -o yaml logger.go:42: 12:03:43 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-verified-possum get pg-backup major-upgrade-logical-replica-reseed-repo1 -o yaml logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-verified-possum get pg-backup major-upgrade-logical-replica-seed-repo1 -o yaml logger.go:42: 12:03:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg resources' logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg resources logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-verified-possum get pg -o json logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + check_operator_panic logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ get_operator_pod logger.go:42: 12:03:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ kubectl get pods -n pg-operator --selector=app.kubernetes.io/name=percona-postgresql-operator -o 'jsonpath={.items[].metadata.name}' logger.go:42: 12:03:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ echo percona-postgresql-operator-6596486d58-zfq6z logger.go:42: 12:03:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local operator_pod=percona-postgresql-operator-6596486d58-zfq6z logger.go:42: 12:03:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local panic_log logger.go:42: 12:03:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl logs -n pg-operator percona-postgresql-operator-6596486d58-zfq6z -c operator logger.go:42: 12:03:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ grep -A 100 'Observed a panic' logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ true logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + panic_log= logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + '[' -n '' ']' logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + destroy_operator logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ -n pg-operator ]] logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 12:03:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 12:03:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | namespace "pg-operator" force deleted logger.go:42: 12:04:01 | major-upgrade-logical-replica/99-remove-cluster-gracefully | test step completed 99-remove-cluster-gracefully logger.go:42: 12:04:02 | major-upgrade-logical-replica | major-upgrade-logical-replica events from ns kuttl-test-verified-possum: logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:34 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-d7c7bb45b from 0 to 1 deployment-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:35 +0000 UTC Normal Pod pg-client-d7c7bb45b-gljfp Scheduled Successfully assigned kuttl-test-verified-possum/pg-client-d7c7bb45b-gljfp to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-vqr0 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:35 +0000 UTC Normal Pod pg-client-d7c7bb45b-gljfp.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:35 +0000 UTC Normal Pod pg-client-d7c7bb45b-gljfp.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 160ms (160ms including waiting). Image size: 445181537 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:35 +0000 UTC Normal Pod pg-client-d7c7bb45b-gljfp.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:35 +0000 UTC Normal Pod pg-client-d7c7bb45b-gljfp.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:35 +0000 UTC Normal ReplicaSet.apps pg-client-d7c7bb45b SuccessfulCreate Created pod: pg-client-d7c7bb45b-gljfp replicaset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:45 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-rlbt-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:45 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-rlbt SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-rlbt-0 in StatefulSet major-upgrade-logical-replica-instance1-rlbt successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:45 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-wjxz-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:46 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-rlbt-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:46 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-rlbt-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-verified-possum/major-upgrade-logical-replica-instance1-rlbt-pgdata" pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:46 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-wjxz-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:46 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-wjxz-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-verified-possum/major-upgrade-logical-replica-instance1-wjxz-pgdata" pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:46 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-wjxz SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-wjxz-0 in StatefulSet major-upgrade-logical-replica-instance1-wjxz successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:47 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-rlbt-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-10b169cf-d648-454c-8265-25c4451fc09a pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:47 +0000 UTC Warning Pod major-upgrade-logical-replica-repo-host-0 FailedScheduling 0/3 nodes are available: persistentvolumeclaim "major-upgrade-logical-replica-repo1" not found. not found default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:47 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulCreate create Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:47 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 WaitForPodScheduled waiting for pod major-upgrade-logical-replica-repo-host-0 to be scheduled persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:47 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:47 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-verified-possum/major-upgrade-logical-replica-repo1" pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:47 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica RepoHostCreated created pgBackRest repository host StatefulSet/major-upgrade-logical-replica-repo-host postgrescluster-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-instance1-rlbt-0 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-7m7j default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-instance1-wjxz-0 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-vqr0 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:48 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-wjxz-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-b7a39d92-19c4-47cf-b1e9-205cb5ef8aea pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:48 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-j489 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:48 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-5fd79dbc87 SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j replicaset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:48 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-5fd79dbc87 from 0 to 1 deployment-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:49 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:49 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 127ms (127ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:49 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:49 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:49 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:49 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-j489 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:49 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-9f825946-0af4-4020-bc44-9d6bfeadeb69 pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:52 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 2.466s (2.466s including waiting). Image size: 84606359 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:52 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:52 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:52 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:52 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 127ms (127ms including waiting). Image size: 84606359 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:52 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:52 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:53 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-10b169cf-d648-454c-8265-25c4451fc09a" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 134ms (134ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b7a39d92-19c4-47cf-b1e9-205cb5ef8aea" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:55 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 166ms (166ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:56 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-9f825946-0af4-4020-bc44-9d6bfeadeb69" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:53:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:00 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:05 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 4.686s (4.686s including waiting). Image size: 122687026 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:05 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:05 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:08 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 149ms (149ms including waiting). Image size: 122687026 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 125ms (125ms including waiting). Image size: 122687026 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 112ms (112ms including waiting). Image size: 122687026 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:12 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 16.807s (16.807s including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:12 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:12 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 16.385s (16.385s including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 125ms (125ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 112ms (112ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 143ms (143ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 113ms (113ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 144ms (144ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{database} Started Started container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 131ms (131ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 122ms (122ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Started Started container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 130ms (130ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{logs} Pulling Pulling image "docker.io/perconalab/fluentbit:main-logcollector" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:30 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-03 11:54:30.583 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-archive-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-09-03 11:54:30.869 P00 ERROR: [055]: unable to load info file '/pgbackrest/repo1/archive/db/archive.info' or '/pgbackrest/repo1/archive/db/archive.info.copy': FileMissingError: unable to open missing file '/pgbackrest/repo1/archive/db/archive.info' for read FileMissingError: unable to open missing file '/pgbackrest/repo1/archive/db/archive.info.copy' for read HINT: archive.info cannot be opened but is required to push/get WAL segments. HINT: is archive_command configured correctly in postgresql.conf? HINT: has a stanza-create been performed? HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme. : command terminated with exit code 55 postgrescluster-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:31 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:34 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:35 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-backup-dhk7-nc9zp to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-7m7j default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:35 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:35 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-dhk7 SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-dhk7-nc9zp job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:36 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 228ms (228ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:36 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:36 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 108ms (108ms including waiting). Image size: 122687026 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-dhk7-nc9zp.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:54:51 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-dhk7 Completed Job completed job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-backup-cmmj-cp7st to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-vqr0 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:00 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-cmmj SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-cmmj-cp7st job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:01 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:01 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 136ms (136ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:01 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:01 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:03 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:03 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 118ms (118ms including waiting). Image size: 122687026 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:03 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:55:03 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cmmj-cp7st.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:56:54 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-cmmj Completed Job completed job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:19 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:19 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:20 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:20 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-verified-possum/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:21 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-20fcafae-6788-40cc-9df9-2113a82ab1a4 pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:22 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-vqr0 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:27 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-20fcafae-6788-40cc-9df9-2113a82ab1a4" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 140ms (140ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:32 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:32 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 113ms (113ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:32 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:32 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:33 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:33 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 115ms (115ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:33 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:33 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-vn5vt.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:45 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:46 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-7m7j default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:46 +0000 UTC Warning Pod major-upgrade-logical-replica-lr-analytics-0 FailedAttachVolume Multi-Attach error for volume "pvc-20fcafae-6788-40cc-9df9-2113a82ab1a4" Volume is already exclusively attached to one node and can't be attached to another attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:57:46 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-20fcafae-6788-40cc-9df9-2113a82ab1a4" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 175ms (175ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 158ms (158ms including waiting). Image size: 447510183 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:03 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:35 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:35 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulDelete delete Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:36 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-rlbt SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-rlbt-0 in StatefulSet major-upgrade-logical-replica-instance1-rlbt successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:38 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:38 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:38 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-5fd79dbc87 SuccessfulDelete Deleted pod: major-upgrade-logical-replica-pgbouncer-5fd79dbc87-9p79j replicaset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:38 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled down replica set major-upgrade-logical-replica-pgbouncer-5fd79dbc87 from 1 to 0 deployment-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:43 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-wjxz SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-wjxz-0 in StatefulSet major-upgrade-logical-replica-instance1-wjxz successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:45 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-logical-replica-set-instance1 NoPods No matching pods found controllermanager logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:48 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulDelete delete Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:50 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b Scheduled Successfully assigned kuttl-test-verified-possum/17-to-18-pgdata-4bt5b to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-7m7j default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:50 +0000 UTC Warning Pod 17-to-18-pgdata-4bt5b FailedAttachVolume Multi-Attach error for volume "pvc-b7a39d92-19c4-47cf-b1e9-205cb5ef8aea" Volume is already exclusively attached to one node and can't be attached to another attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:58:50 +0000 UTC Normal Job.batch 17-to-18-pgdata SuccessfulCreate Created pod: 17-to-18-pgdata-4bt5b job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:01 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b7a39d92-19c4-47cf-b1e9-205cb5ef8aea" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:02 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:03 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 129ms (129ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:03 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:03 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:03 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:37 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 33.664s (33.664s including waiting). Image size: 952997237 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:37 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b.spec.containers{database} Created Created container: database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:37 +0000 UTC Normal Pod 17-to-18-pgdata-4bt5b.spec.containers{database} Started Started container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:54 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-rlbt-qkvfx Scheduled Successfully assigned kuttl-test-verified-possum/17-to-18-major-upgrade-logical-replica-instance1-rlbt-qkvfx to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-7m7j default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:54 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-rlbt SuccessfulCreate Created pod: 17-to-18-major-upgrade-logical-replica-instance1-rlbt-qkvfx job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:54 +0000 UTC Normal Job.batch 17-to-18-pgdata Completed Job completed job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 11:59:59 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-rlbt-qkvfx SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-10b169cf-d648-454c-8265-25c4451fc09a" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:02 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-rlbt-qkvfx.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:02 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-rlbt-qkvfx.spec.containers{database} Created Created container: database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:02 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-rlbt-qkvfx.spec.containers{database} Started Started container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:05 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-rlbt Completed Job completed job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:12 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-instance1-wjxz-0 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-7m7j default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:13 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-j489 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:13 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-58ffbd7c8c SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5 replicaset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:13 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-58ffbd7c8c from 0 to 1 deployment-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:13 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-vqr0 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:14 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:14 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 146ms (146ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:14 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:14 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 102ms (102ms including waiting). Image size: 84619986 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 140ms (140ms including waiting). Image size: 84619986 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:19 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-9f825946-0af4-4020-bc44-9d6bfeadeb69" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b7a39d92-19c4-47cf-b1e9-205cb5ef8aea" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 157ms (158ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 176ms (176ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 125ms (126ms including waiting). Image size: 122700418 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 111ms (112ms including waiting). Image size: 122700418 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:24 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 120ms (120ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 120ms (120ms including waiting). Image size: 122700418 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:25 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 122ms (122ms including waiting). Image size: 122700418 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 104ms (104ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:26 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:26 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 112ms (112ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{database} Started Started container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 119ms (119ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-wjxz-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:35 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-instance1-rlbt-0 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-j489 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-backup-k7nc-jxnwd to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-vqr0 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 138ms (138ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:38 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-k7nc SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-k7nc-jxnwd job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 154ms (154ms including waiting). Image size: 122700418 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-k7nc-jxnwd.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:40 +0000 UTC Warning PerconaPGCluster.pgv2.percona.com major-upgrade-logical-replica LogicalReplicaInvalidated Logical replica "analytics" must be seeded again: the cluster was upgraded from PostgreSQL 17 to 18 after it was seeded perconapgcluster-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-10b169cf-d648-454c-8265-25c4451fc09a" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:43 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:44 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 181ms (181ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:44 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:44 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:45 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:00:50 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-k7nc Completed Job completed job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 16.302s (16.302s including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 134ms (134ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 125ms (125ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 117ms (117ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{database} Started Started container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 124ms (124ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-rlbt-0.spec.containers{logs} Pulling Pulling image "docker.io/perconalab/fluentbit:main-logcollector" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:44 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-backup-cd7k-bc6z8 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-vqr0 default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:44 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-cd7k SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-cd7k-bc6z8 job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:45 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:45 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 267ms (267ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:45 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:45 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:46 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:47 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 341ms (341ms including waiting). Image size: 122700418 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:47 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:01:47 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-cd7k-bc6z8.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:02 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-cd7k Completed Job completed job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:33 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:34 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:34 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-verified-possum/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:36 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-7m7j default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:36 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-ad2b4681-3916-4032-acfe-1caa14f1cb93 pd.csi.storage.gke.io_gke-d275382c2d004b19bdc7-3337-5e21-vm_963658c3-557d-4af8-b091-2573bafa43ca logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:42 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-ad2b4681-3916-4032-acfe-1caa14f1cb93" attachdetach-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:46 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:46 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1814-d67db8e10" in 247ms (247ms including waiting). Image size: 87058745 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:46 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:46 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:47 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 211ms (211ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 217ms (217ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:02:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-7zqrm.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:01 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Scheduled Successfully assigned kuttl-test-verified-possum/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1814-d67db8e1-default-pool-e7fc58a1-7m7j default-scheduler logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:02 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:06 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:06 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 147ms (147ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:06 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:07 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:07 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:08 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 172ms (172ms including waiting). Image size: 449178524 bytes. kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:08 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:08 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:33 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:40 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:40 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-58ffbd7c8c-8vzk5.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:40 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | 2026-09-03 12:03:40 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 12:04:02 | major-upgrade-logical-replica | Deleting namespace "kuttl-test-verified-possum" === NAME kuttl harness.go:404: run tests finished harness.go:511: cleaning up harness.go:568: removing temp folder: "" --- PASS: kuttl (713.16s) --- PASS: kuttl/harness (0.00s) --- PASS: kuttl/harness/major-upgrade-logical-replica (712.22s) PASS