=== RUN kuttl harness.go:469: starting setup harness.go:259: running tests using configured kubeconfig. harness.go:282: Successful connection to cluster at: https://136.111.39.211 harness.go:365: running tests harness.go:78: going to run test suite with timeout of 600 seconds for each step harness.go:377: testsuite: e2e-tests/tests has 60 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-logical-replica === PAUSE kuttl/harness/major-upgrade-logical-replica === CONT kuttl/harness/major-upgrade-logical-replica logger.go:42: 00:13:57 | major-upgrade-logical-replica | Ignoring "conf": does not begin with a number followed by a dash. logger.go:42: 00:13:57 | major-upgrade-logical-replica | Creating namespace "kuttl-test-darling-moose" logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | + source ../../functions logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ realpath ../../.. logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++ BUSYBOX_VER=1.36 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++++ pwd logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++ test_name=major-upgrade-logical-replica logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export GIT_BRANCH=PR-1812 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ GIT_BRANCH=PR-1812 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export VERSION=PR-1812-ee83a33b2 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ VERSION=PR-1812-ee83a33b2 logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -z '' ]] logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++++ detect_k8s_platform logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++++ local detected_platform logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++++ local xtrace_enabled=false logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++++ [[ -o xtrace ]] logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++++ xtrace_enabled=true logger.go:42: 00:13:58 | major-upgrade-logical-replica/0-deploy-operator | ++++ set +o xtrace logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ PLATFORM=gke logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export PLATFORM logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_VER=18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_DISTRIBUTION= logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_DISTRIBUTION= logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ '' == community ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export CPGO_VERSION=5.8.7 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ CPGO_VERSION=5.8.7 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export VAULT_VER=0.32.0 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ VAULT_VER=0.32.0 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ printenv logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gdate logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ which date logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gsed logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | ++++ which sed logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + init_temp_dir logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + deploy_operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + local cw_prefix= logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + destroy_operator logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | Error from server (NotFound): deployments.apps "percona-postgresql-operator" not found logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + true logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 00:14:00 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 00:14:01 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 00:14:01 | major-upgrade-logical-replica/0-deploy-operator | + create_namespace pg-operator logger.go:42: 00:14:01 | major-upgrade-logical-replica/0-deploy-operator | + local namespace=pg-operator logger.go:42: 00:14:01 | major-upgrade-logical-replica/0-deploy-operator | + [[ gke == openshift ]] logger.go:42: 00:14:01 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 00:14:01 | major-upgrade-logical-replica/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 00:14:02 | major-upgrade-logical-replica/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 00:14:02 | major-upgrade-logical-replica/0-deploy-operator | namespace/pg-operator created logger.go:42: 00:14:02 | major-upgrade-logical-replica/0-deploy-operator | + cw_prefix=cw- logger.go:42: 00:14:02 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/crd.yaml logger.go:42: 00:14:03 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 00:14:03 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 00:14:05 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 00:14:05 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 00:14:05 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 00:14:05 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.upstream.pgv2.percona.com serverside-applied logger.go:42: 00:14:05 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.upstream.pgv2.percona.com serverside-applied logger.go:42: 00:14:07 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 00:14:07 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/cw-rbac.yaml logger.go:42: 00:14:07 | major-upgrade-logical-replica/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | + local disable_telemetry=true logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | + '[' major-upgrade-logical-replica == telemetry-transfer ']' logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/cw-operator.yaml logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 00:14:08 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 00:14:09 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 00:14:09 | major-upgrade-logical-replica/0-deploy-operator | + deploy_client logger.go:42: 00:14:09 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n kuttl-test-darling-moose apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf/client.yaml logger.go:42: 00:14:10 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 00:14:10 | major-upgrade-logical-replica/0-deploy-operator | + deploy_s3_secrets logger.go:42: 00:14:10 | major-upgrade-logical-replica/0-deploy-operator | + set +o xtrace logger.go:42: 00:14:10 | major-upgrade-logical-replica/0-deploy-operator | secret/major-upgrade-logical-replica-pgbackrest-secrets created logger.go:42: 00:14:11 | major-upgrade-logical-replica/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 00:14:12 | major-upgrade-logical-replica/0-deploy-operator | secret/do-spaces-secret created logger.go:42: 00:14:12 | major-upgrade-logical-replica/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 00:14:12 | major-upgrade-logical-replica/0-deploy-operator | secret/azure-secret created logger.go:42: 00:14:13 | major-upgrade-logical-replica/0-deploy-operator | secret/minio-secret created logger.go:42: 00:14:14 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 00:14:14 | major-upgrade-logical-replica/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 00:14:15 | major-upgrade-logical-replica/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 00:14:15 | major-upgrade-logical-replica/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 00:14:15 | major-upgrade-logical-replica/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 00:14:15 | major-upgrade-logical-replica/0-deploy-operator | ASSERT PASS logger.go:42: 00:14:15 | major-upgrade-logical-replica/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | starting test step 1-create-cluster logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions postgres_image=$(get_container_image "postgres" 17) pgbouncer_image=$(get_container_image "pgbouncer" 17) pgbackrest_image=$(get_container_image "pgbackrest" 17) get_cr \ | yq eval " .spec.postgresVersion = 17 | .spec.image = \"${postgres_image}\" | .spec.proxy.pgBouncer.image = \"${pgbouncer_image}\" | .spec.backups.pgbackrest.image = \"${pgbackrest_image}\" | .spec.patroni.removeDataDirectoryOnDivergedTimelines = true" \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | + source ../../functions logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ realpath ../../.. logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++ BUSYBOX_VER=1.36 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++++ pwd logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++ test_name=major-upgrade-logical-replica logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export GIT_BRANCH=PR-1812 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ GIT_BRANCH=PR-1812 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export VERSION=PR-1812-ee83a33b2 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ VERSION=PR-1812-ee83a33b2 logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -z '' ]] logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++++ detect_k8s_platform logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++++ local detected_platform logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++++ local xtrace_enabled=false logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++++ [[ -o xtrace ]] logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++++ xtrace_enabled=true logger.go:42: 00:14:15 | major-upgrade-logical-replica/1-create-cluster | ++++ set +o xtrace logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ PLATFORM=gke logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export PLATFORM logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_VER=18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ PG_VER=18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_DISTRIBUTION= logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ PG_DISTRIBUTION= logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ '' == community ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export CPGO_VERSION=5.8.7 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ CPGO_VERSION=5.8.7 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export VAULT_VER=0.32.0 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ VAULT_VER=0.32.0 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ printenv logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ which gdate logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ which date logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ which gsed logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++++ which sed logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image postgres 17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local component=postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1812-ee83a33b2 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbouncer ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbackrest ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == upgrade ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == postgis ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + postgres_image=docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbouncer 17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbouncer logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1812-ee83a33b2 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbouncer == pgbouncer ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + pgbouncer_image=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbackrest 17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbackrest logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/percona/* ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1812-ee83a33b2 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1812-ee83a33b2 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbouncer ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbackrest ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + pgbackrest_image=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + get_cr logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + local cr_name=major-upgrade-logical-replica logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + local repo_path= logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + local source_path= logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + local platform=gke logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + local cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + local platform_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + local spaces_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-spaces.yaml logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == doks ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + crs=('/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/cr.yaml') logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + local crs logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + crs+=($cr_file) logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + yq eval-all $'\n\t\tselect(fileIndex == 0) * (select(fileIndex == 1) // {}) |\n\t\t.metadata.name = "major-upgrade-logical-replica" |\n\t\t.metadata.labels = {"e2e":"major-upgrade-logical-replica"} |\n\t\t.spec.postgresVersion = 18 |\n\t\t.spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.users += [{"name":"major-upgrade-logical-replica","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" |\n\t\t.spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" |\n\t\t.spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" |\n\t\t.spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" |\n\t\t.spec.logcollector.image = "docker.io/perconalab/fluentbit:main-logcollector" |\n\t\t.spec.pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest" |\n\t\t.spec.pmm.secret = "major-upgrade-logical-replica-pmm-secret" |\n\t\t.spec.pmm.customClusterName = "major-upgrade-logical-replica-pmm-custom-name" |\n\t\t.spec.pmm.postgresParams = "--environment=dev-postgres"\n\t' /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/cr.yaml /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + yq eval $'\n .spec.postgresVersion = 17 |\n .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" |\n .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" |\n .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" |\n .spec.patroni.removeDataDirectoryOnDivergedTimelines = true' logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + kubectl -n kuttl-test-darling-moose apply -f - logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||pg-operator-testing|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == openshift ]] logger.go:42: 00:14:17 | major-upgrade-logical-replica/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 00:14:18 | major-upgrade-logical-replica/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica created logger.go:42: 00:15:12 | major-upgrade-logical-replica/1-create-cluster | test step completed 1-create-cluster logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | starting test step 2-write-data logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:${PASS}@${HOST}" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:${PASS}@${HOST}"] logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | + source ../../functions logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ realpath ../../.. logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++ BUSYBOX_VER=1.36 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++++ pwd logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++ test_name=major-upgrade-logical-replica logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export GIT_BRANCH=PR-1812 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ GIT_BRANCH=PR-1812 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export VERSION=PR-1812-ee83a33b2 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ VERSION=PR-1812-ee83a33b2 logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | +++ [[ -z '' ]] logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++++ detect_k8s_platform logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++++ local detected_platform logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++++ local xtrace_enabled=false logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++++ [[ -o xtrace ]] logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++++ xtrace_enabled=true logger.go:42: 00:15:12 | major-upgrade-logical-replica/2-write-data | ++++ set +o xtrace logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ PLATFORM=gke logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export PLATFORM logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ -n '' ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export PG_VER=18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ PG_VER=18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export PG_DISTRIBUTION= logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ PG_DISTRIBUTION= logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ '' == community ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_VER=14 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export CPGO_VERSION=5.8.7 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ CPGO_VERSION=5.8.7 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export VAULT_VER=0.32.0 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ VAULT_VER=0.32.0 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ printenv logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ which gdate logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ which date logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ date=/usr/sbin/date logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ which gsed logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++++ which sed logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-darling-moose get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | + PASS=a9LYzODviIfqAHQ8gUI61PlI logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:15:14 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-darling-moose get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 00:15:15 | major-upgrade-logical-replica/2-write-data | + HOST=major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:15:15 | major-upgrade-logical-replica/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:15:15 | major-upgrade-logical-replica/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 00:15:15 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:15:15 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 00:15:15 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 00:15:15 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-darling-moose get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:15:15 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-darling-moose exec pg-client-d7c7bb45b-2l4vf -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc'\''' logger.go:42: 00:15:16 | major-upgrade-logical-replica/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:15:16 | major-upgrade-logical-replica/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 00:15:16 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:15:16 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 00:15:16 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 00:15:16 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-darling-moose get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:15:17 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-darling-moose exec pg-client-d7c7bb45b-2l4vf -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc'\''' logger.go:42: 00:15:18 | major-upgrade-logical-replica/2-write-data | test step completed 2-write-data logger.go:42: 00:15:18 | major-upgrade-logical-replica/3-create-seed-backup | starting test step 3-create-seed-backup logger.go:42: 00:15:18 | major-upgrade-logical-replica/3-create-seed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # The pgbackrest bootstrap Job restores the latest full backup, and the # one taken at cluster creation predates the rows written above. Take a # new one so the seeded data is actually in the backup being restored. cat </dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:45 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:47 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:49 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:51 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:52 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:54 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:56 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:58 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:18:59 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:01 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:03 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:04 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:05 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:06 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:08 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:09 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:09 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:12 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:13 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:15 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:15 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:16 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:18 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:19 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:20 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:22 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:22 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:23 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:25 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:26 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:27 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:29 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:29 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:30 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:32 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:33 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:34 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:36 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:36 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:37 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:39 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:40 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:41 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:43 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:43 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:44 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:46 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:47 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:47 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qw4m4" is waiting to start: PodInitializing logger.go:42: 00:19:50 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:50 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:19:51 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:19:53 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:19:54 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:19:57 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:19:57 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:19:58 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:00 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:01 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:02 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:04 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:04 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:05 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:07 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:08 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:09 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:11 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:12 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:14 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:15 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:16 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:18 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:18 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:19 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:21 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:22 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:23 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:25 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:25 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:26 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:28 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:29 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:30 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:32 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:32 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:33 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:35 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:36 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:39 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:39 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:40 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:43 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:44 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:46 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:46 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:47 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:49 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:50 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:53 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:53 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:54 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:56 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:20:57 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:20:59 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:00 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:01 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:03 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:04 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:06 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:07 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:08 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:10 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:11 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:13 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:14 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:15 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:17 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:17 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:18 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:20 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:21 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:22 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:24 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:24 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:25 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:27 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:28 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:29 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:31 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:31 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:32 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:34 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:35 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:36 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:21:38 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | real 0m1.892s logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | user 0m0.077s logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | sys 0m0.103s logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | real 0m9.411s logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | user 0m0.254s logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | sys 0m0.411s logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:21:39 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:21:40 | major-upgrade-logical-replica/5-upgrade | test step completed 5-upgrade logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | starting test step 6-verify-replica-invalidated logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-logical-replica primary name) kubectl -n "${NAMESPACE}" exec "${primary}" -- /pgdata/delete_old_cluster.sh # pg_upgrade only rewrites the data directories of the instance sets, so the # replica still holds a PostgreSQL 17 one. It has to be reported as broken # rather than handed the new binary and left to crash. retry 60 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") reason=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].reason}") echo "logical replica analytics: ${state}/${reason}" [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]] ' message=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].message}') echo "message: ${message}" echo "${message}" | grep -q "remove it from spec.logicalReplicas" # The StatefulSet is stopped, so nothing is left crash-looping against a data # directory the new binary cannot open. retry 30 5 bash -c ' replicas=$(kubectl -n "'"${NAMESPACE}"'" get statefulset \ major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}") echo "logical replica statefulset replicas: ${replicas}" [[ ${replicas} == "0" ]] ' # pg_upgrade carries the logical slots over to the new cluster. Nothing will # ever read from them again, so they must not be left pinning WAL. PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) no_slots_left() { local got got=$(run_psql \ "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ "postgres://postgres:${PASS}@${HOST}" | xargs) echo "logical slots left on the primary: ${got}" [[ ${got} == "0" ]] } retry 30 5 no_slots_left] logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | + source ../../functions logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ realpath ../../.. logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ BUSYBOX_VER=1.36 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ pwd logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ test_name=major-upgrade-logical-replica logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export GIT_BRANCH=PR-1812 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ GIT_BRANCH=PR-1812 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VERSION=PR-1812-ee83a33b2 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VERSION=PR-1812-ee83a33b2 logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ SKIP_TEST_WARNINGS=true logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -z '' ]] logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ detect_k8s_platform logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local detected_platform logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local xtrace_enabled=false logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ [[ -o xtrace ]] logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ xtrace_enabled=true logger.go:42: 00:21:40 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ set +o xtrace logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PLATFORM=gke logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PLATFORM logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ ! -n '' ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -n '' ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_VER=18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_VER=18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_DISTRIBUTION= logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_DISTRIBUTION= logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ '' == community ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export BUCKET=pg-operator-testing logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ BUCKET=pg-operator-testing logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_TAG=1.4.0 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_TAG=1.4.0 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_VER=14 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_VER=14 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export CPGO_VERSION=5.8.7 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ CPGO_VERSION=5.8.7 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export MINIO_VER=5.4.0 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ MINIO_VER=5.4.0 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VAULT_VER=0.32.0 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VAULT_VER=0.32.0 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME=docker.io logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME=docker.io logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ printenv logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ grep -E '^IMAGE' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ awk -F= '{print $1}' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gdate logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which date logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ date=/usr/sbin/date logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gsed logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which sed logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ sed=/usr/sbin/sed logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_pod_by_role major-upgrade-logical-replica primary name logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local cluster=major-upgrade-logical-replica logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local role=primary logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local parameter=name logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ case ${parameter} in logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'jsonpath={.items[].metadata.name}' logger.go:42: 00:21:42 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl get pods --namespace kuttl-test-darling-moose --selector=postgres-operator.crunchydata.com/role=primary,postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:21:43 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ echo major-upgrade-logical-replica-instance1-q8vs-0 logger.go:42: 00:21:43 | major-upgrade-logical-replica/6-verify-replica-invalidated | + primary=major-upgrade-logical-replica-instance1-q8vs-0 logger.go:42: 00:21:43 | major-upgrade-logical-replica/6-verify-replica-invalidated | + kubectl -n kuttl-test-darling-moose exec major-upgrade-logical-replica-instance1-q8vs-0 -- /pgdata/delete_old_cluster.sh logger.go:42: 00:21:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 60 10 bash -c $'\n state=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 00:21:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=60 logger.go:42: 00:21:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=10 logger.go:42: 00:21:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 00:21:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 00:21:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n state=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica analytics: broken/SourceUpgraded logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-darling-moose get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].message}' logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + message='this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + grep -q 'remove it from spec.logicalReplicas' logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 bash -c $'\n replicas=$(kubectl -n "kuttl-test-darling-moose" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 00:21:45 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n replicas=$(kubectl -n "kuttl-test-darling-moose" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 00:21:46 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica statefulset replicas: 0 logger.go:42: 00:21:46 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:21:46 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:21:46 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-darling-moose get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 00:21:46 | major-upgrade-logical-replica/6-verify-replica-invalidated | + PASS=a9LYzODviIfqAHQ8gUI61PlI logger.go:42: 00:21:46 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:21:46 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:21:46 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-darling-moose get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | + HOST=major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 no_slots_left logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | + no_slots_left logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local got logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ run_psql 'SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ xargs logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'command=SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local uri=postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local password= logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ get_client_pod logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl -n kuttl-test-darling-moose get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:21:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-darling-moose exec pg-client-d7c7bb45b-2l4vf -- bash -c 'printf '\''SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc' logger.go:42: 00:21:48 | major-upgrade-logical-replica/6-verify-replica-invalidated | + got=0 logger.go:42: 00:21:48 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'logical slots left on the primary: 0' logger.go:42: 00:21:48 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical slots left on the primary: 0 logger.go:42: 00:21:48 | major-upgrade-logical-replica/6-verify-replica-invalidated | + [[ 0 == 0 ]] logger.go:42: 00:21:48 | major-upgrade-logical-replica/6-verify-replica-invalidated | test step completed 6-verify-replica-invalidated logger.go:42: 00:21:48 | major-upgrade-logical-replica/7-create-reseed-backup | starting test step 7-create-reseed-backup logger.go:42: 00:21:48 | major-upgrade-logical-replica/7-create-reseed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # pg_upgrade left the repository holding only pre-upgrade backups, so the # bootstrap Job of the reseed below would restore a PostgreSQL 17 data # directory. Take a full backup of the upgraded cluster first. cat </dev/null 2>&1 && return 1 kubectl -n "${NAMESPACE}" get pvc major-upgrade-logical-replica-lr-analytics-pgdata >/dev/null 2>&1 && return 1 return 0 } retry 60 5 status_gone retry 60 5 objects_gone kubectl -n "${NAMESPACE}" patch pg major-upgrade-logical-replica --type=merge --patch '{ "spec": { "logicalReplicas": [ { "name": "analytics", "databases": ["myapp"], "bootstrapMethod": "pgbackrest", "dataVolumeClaimSpec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}} } } ] } }'] logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1812 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1812 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1812-ee83a33b2 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1812-ee83a33b2 logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 00:23:37 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 00:23:39 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-darling-moose patch pg major-upgrade-logical-replica --type=json --patch '[{"op": "remove", "path": "/spec/logicalReplicas"}]' logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 status_gone logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-darling-moose get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + got='[{"invalidatedAt":"2026-09-04T00:21:17Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-04T00:18:20Z","state":"broken"}]' logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\''[{"invalidatedAt":"2026-09-04T00:21:17Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-04T00:18:20Z","state":"broken"}]'\''' logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '[{"invalidatedAt":"2026-09-04T00:21:17Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-04T00:18:20Z","state":"broken"}]' logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z [{"invalidatedAt":"2026-09-04T00:21:17Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-04T00:18:20Z","state":"broken"}] ]] logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ [{"invalidatedAt":"2026-09-04T00:21:17Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-04T00:18:20Z","state":"broken"}] == \[\] ]] logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 60 ]] logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 00:23:40 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 5 logger.go:42: 00:23:45 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 00:23:45 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 00:23:45 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-darling-moose get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + got= logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\'''\''' logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '' logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z '' ]] logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 objects_gone logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + objects_gone logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-darling-moose get statefulset major-upgrade-logical-replica-lr-analytics logger.go:42: 00:23:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-darling-moose get pvc major-upgrade-logical-replica-lr-analytics-pgdata logger.go:42: 00:23:47 | major-upgrade-logical-replica/8-reseed-logical-replica | + return 0 logger.go:42: 00:23:47 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-darling-moose patch pg major-upgrade-logical-replica --type=merge --patch $'{\n "spec": {\n "logicalReplicas": [\n {\n "name": "analytics",\n "databases": ["myapp"],\n "bootstrapMethod": "pgbackrest",\n "dataVolumeClaimSpec": {\n "accessModes": ["ReadWriteOnce"],\n "resources": {"requests": {"storage": "1Gi"}}\n }\n }\n ]\n }\n}' logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | running command: [sh -c set -o errexit set -o xtrace source ../../functions retry 90 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") echo "logical replica analytics state: ${state}" [[ ${state} == "ready" ]] ' # Seeded from the upgraded cluster, so it is stamped with the new version. seeded=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}') echo "seeded at PostgreSQL: ${seeded}" [[ ${seeded} == "18" ]] kubectl -n "${NAMESPACE}" wait --for=condition=Ready pod \ -l "pgv2.percona.com/logical-replica=analytics" --timeout=300s PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) REPLICA="major-upgrade-logical-replica-lr-analytics.${NAMESPACE}.svc" # Replication actually streams again on the upgraded cluster. run_psql \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' \ "postgres://postgres:${PASS}@${HOST}" row_streamed() { local got got=$(run_psql \ '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' \ "postgres://postgres:${PASS}@${REPLICA}" | xargs) echo "streamed rows on the replica: ${got}" [[ ${got} == "1" ]] } retry 30 5 row_streamed] logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1812 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1812 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1812-ee83a33b2 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1812-ee83a33b2 logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 00:23:48 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 90 10 bash -c $'\n state=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=90 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=10 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 90 ]] logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 00:23:50 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 00:24:00 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 00:24:01 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 00:24:01 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 2 -ge 90 ]] logger.go:42: 00:24:01 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 00:24:01 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 00:24:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 00:24:11 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 00:24:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 3 -ge 90 ]] logger.go:42: 00:24:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 00:24:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 00:24:21 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 00:24:22 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 00:24:22 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 4 -ge 90 ]] logger.go:42: 00:24:22 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 00:24:22 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 00:24:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-darling-moose" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 00:24:32 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: ready logger.go:42: 00:24:32 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-darling-moose get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}' logger.go:42: 00:24:33 | major-upgrade-logical-replica/8-reseed-logical-replica | + seeded=18 logger.go:42: 00:24:33 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'seeded at PostgreSQL: 18' logger.go:42: 00:24:33 | major-upgrade-logical-replica/8-reseed-logical-replica | seeded at PostgreSQL: 18 logger.go:42: 00:24:33 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 18 == 18 ]] logger.go:42: 00:24:33 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-darling-moose wait --for=condition=Ready pod -l pgv2.percona.com/logical-replica=analytics --timeout=300s logger.go:42: 00:24:34 | major-upgrade-logical-replica/8-reseed-logical-replica | pod/major-upgrade-logical-replica-lr-analytics-0 condition met logger.go:42: 00:24:34 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:24:34 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:24:34 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-darling-moose get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 00:24:34 | major-upgrade-logical-replica/8-reseed-logical-replica | + PASS=a9LYzODviIfqAHQ8gUI61PlI logger.go:42: 00:24:34 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:24:34 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:24:34 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-darling-moose get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + HOST=major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + REPLICA=major-upgrade-logical-replica-lr-analytics.kuttl-test-darling-moose.svc logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + local uri=postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + local password= logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_client_pod logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-darling-moose get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:24:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-darling-moose exec pg-client-d7c7bb45b-2l4vf -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-primary.kuttl-test-darling-moose.svc' logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 30 5 row_streamed logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=30 logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + row_streamed logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ run_psql '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-lr-analytics.kuttl-test-darling-moose.svc logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ xargs logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local 'command=\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local uri=postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-lr-analytics.kuttl-test-darling-moose.svc logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local password= logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ get_client_pod logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ kubectl -n kuttl-test-darling-moose get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:24:36 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-darling-moose exec pg-client-d7c7bb45b-2l4vf -- bash -c 'printf '\''\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:a9LYzODviIfqAHQ8gUI61PlI@major-upgrade-logical-replica-lr-analytics.kuttl-test-darling-moose.svc' logger.go:42: 00:24:37 | major-upgrade-logical-replica/8-reseed-logical-replica | + got=1 logger.go:42: 00:24:37 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'streamed rows on the replica: 1' logger.go:42: 00:24:37 | major-upgrade-logical-replica/8-reseed-logical-replica | streamed rows on the replica: 1 logger.go:42: 00:24:37 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 == 1 ]] logger.go:42: 00:24:37 | major-upgrade-logical-replica/8-reseed-logical-replica | test step completed 8-reseed-logical-replica logger.go:42: 00:24:37 | major-upgrade-logical-replica/99-remove-cluster-gracefully | starting test step 99-remove-cluster-gracefully logger.go:42: 00:24:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | PerconaPGCluster:kuttl-test-darling-moose/major-upgrade-logical-replica deleted logger.go:42: 00:24:39 | major-upgrade-logical-replica/99-remove-cluster-gracefully | PostgresCluster:kuttl-test-darling-moose/major-upgrade-logical-replica deleted logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | running command: [sh -c set -o errexit set -o xtrace source ../../functions remove_all_finalizers check_operator_panic destroy_operator] logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + source ../../functions logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ realpath ../../.. logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ BUSYBOX_VER=1.36 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ pwd logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ test_name=major-upgrade-logical-replica logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export GIT_BRANCH=PR-1812 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ GIT_BRANCH=PR-1812 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VERSION=PR-1812-ee83a33b2 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VERSION=PR-1812-ee83a33b2 logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ SKIP_TEST_WARNINGS=true logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -z '' ]] logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ detect_k8s_platform logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local detected_platform logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local xtrace_enabled=false logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ [[ -o xtrace ]] logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ xtrace_enabled=true logger.go:42: 00:24:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ set +o xtrace logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PLATFORM=gke logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PLATFORM logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ ! -n '' ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -n '' ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_VER=18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_VER=18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_DISTRIBUTION= logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_DISTRIBUTION= logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ '' == community ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export BUCKET=pg-operator-testing logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ BUCKET=pg-operator-testing logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_TAG=1.4.0 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_TAG=1.4.0 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_VER=14 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_VER=14 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export CPGO_VERSION=5.8.7 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ CPGO_VERSION=5.8.7 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export MINIO_VER=5.4.0 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ MINIO_VER=5.4.0 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VAULT_VER=0.32.0 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VAULT_VER=0.32.0 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME=docker.io logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME=docker.io logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ printenv logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ grep -E '^IMAGE' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ awk -F= '{print $1}' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2 logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gdate logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which date logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ date=/usr/sbin/date logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gsed logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which sed logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ sed=/usr/sbin/sed logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + remove_all_finalizers logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + resource_types=("pg-restore" "pg-backup" "pg") logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-restore resources' logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-restore resources logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-darling-moose get pg-restore -o json logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 00:24:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-backup resources' logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-backup resources logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-darling-moose get pg-backup -o json logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-darling-moose get pg-backup major-upgrade-logical-replica-backup-5ll2-tzsln -o yaml logger.go:42: 00:24:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-darling-moose get pg-backup major-upgrade-logical-replica-backup-97bl-l67vd -o yaml logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-darling-moose get pg-backup major-upgrade-logical-replica-reseed-repo1 -o yaml logger.go:42: 00:24:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-darling-moose get pg-backup major-upgrade-logical-replica-seed-repo1 -o yaml logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg resources' logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg resources logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-darling-moose get pg -o json logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 00:24:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 00:24:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + check_operator_panic logger.go:42: 00:24:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ get_operator_pod logger.go:42: 00:24:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ kubectl get pods -n pg-operator --selector=app.kubernetes.io/name=percona-postgresql-operator -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:24:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ echo percona-postgresql-operator-5ddd5459f5-vzqq8 logger.go:42: 00:24:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local operator_pod=percona-postgresql-operator-5ddd5459f5-vzqq8 logger.go:42: 00:24:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local panic_log logger.go:42: 00:24:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl logs -n pg-operator percona-postgresql-operator-5ddd5459f5-vzqq8 -c operator logger.go:42: 00:24:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ grep -A 100 'Observed a panic' logger.go:42: 00:24:51 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ true logger.go:42: 00:24:51 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + panic_log= logger.go:42: 00:24:51 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + '[' -n '' ']' logger.go:42: 00:24:51 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + destroy_operator logger.go:42: 00:24:51 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 00:24:51 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 00:24:52 | major-upgrade-logical-replica/99-remove-cluster-gracefully | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 00:24:52 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ -n pg-operator ]] logger.go:42: 00:24:52 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 00:24:52 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 00:24:52 | major-upgrade-logical-replica/99-remove-cluster-gracefully | namespace "pg-operator" force deleted logger.go:42: 00:25:04 | major-upgrade-logical-replica/99-remove-cluster-gracefully | test step completed 99-remove-cluster-gracefully logger.go:42: 00:25:04 | major-upgrade-logical-replica | major-upgrade-logical-replica events from ns kuttl-test-darling-moose: logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:10 +0000 UTC Normal Pod pg-client-d7c7bb45b-2l4vf Scheduled Successfully assigned kuttl-test-darling-moose/pg-client-d7c7bb45b-2l4vf to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-smm7 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:10 +0000 UTC Normal Pod pg-client-d7c7bb45b-2l4vf.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:10 +0000 UTC Normal ReplicaSet.apps pg-client-d7c7bb45b SuccessfulCreate Created pod: pg-client-d7c7bb45b-2l4vf replicaset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:10 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-d7c7bb45b from 0 to 1 deployment-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:11 +0000 UTC Normal Pod pg-client-d7c7bb45b-2l4vf.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 148ms (148ms including waiting). Image size: 445181537 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:11 +0000 UTC Normal Pod pg-client-d7c7bb45b-2l4vf.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:11 +0000 UTC Normal Pod pg-client-d7c7bb45b-2l4vf.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:20 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-bwv5-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:20 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-q8vs SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-q8vs-0 in StatefulSet major-upgrade-logical-replica-instance1-q8vs successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:20 +0000 UTC Warning Service major-upgrade-logical-replica-replicas ClusterIPAllocation ClusterIPNotAllocated Cluster IP [IPv4]: 34.118.233.244 is not allocated; repairing ipallocator-repair-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:20 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-logical-replica-set-instance1 NoPods No matching pods found controllermanager logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:21 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-bwv5-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:21 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-bwv5-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-darling-moose/major-upgrade-logical-replica-instance1-bwv5-pgdata" pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:21 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-bwv5 SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-bwv5-0 in StatefulSet major-upgrade-logical-replica-instance1-bwv5 successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:21 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-q8vs-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:21 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-q8vs-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:21 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-q8vs-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-darling-moose/major-upgrade-logical-replica-instance1-q8vs-pgdata" pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:22 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulCreate create Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:22 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:22 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica RepoHostCreated created pgBackRest repository host StatefulSet/major-upgrade-logical-replica-repo-host postgrescluster-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-instance1-bwv5-0 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:23 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-bwv5-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-f903415a-4318-44f3-a1f4-70ecfa2e11c9 pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:23 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-q8vs-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-0be6693b-50f1-49e6-8e8d-87eb53e640a3 pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:23 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-jx94 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:23 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-7ff76cfd86 SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws replicaset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:23 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-7ff76cfd86 from 0 to 1 deployment-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:23 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:23 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-darling-moose/major-upgrade-logical-replica-repo1" pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-instance1-q8vs-0 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-smm7 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:24 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:24 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 166ms (166ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:24 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:24 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:24 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-bb6f52a0-003c-4992-aa9b-1d1ad495e6d8 pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:25 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:25 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-jx94 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:28 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-f903415a-4318-44f3-a1f4-70ecfa2e11c9" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 2.592s (2.592s including waiting). Image size: 84606212 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 138ms (139ms including waiting). Image size: 84606212 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:29 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-0be6693b-50f1-49e6-8e8d-87eb53e640a3" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 132ms (132ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 161ms (161ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-bb6f52a0-003c-4992-aa9b-1d1ad495e6d8" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:32 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:37 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 4.446s (4.446s including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:37 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:37 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:40 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:40 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 113ms (113ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:40 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:40 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:41 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:41 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 105ms (105ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:41 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:41 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:41 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:41 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 110ms (110ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:41 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:42 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:47 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 16.056s (16.056s including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:47 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:47 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 16.12s (16.12s including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 186ms (186ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 122ms (122ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 112ms (112ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 119ms (119ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 117ms (117ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Started Started container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 122ms (122ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 140ms (140ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Started Started container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 114ms (114ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:14:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:03 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:04 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-backup-97bl-b9tzq to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-smm7 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:04 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:04 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 196ms (196ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:04 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:04 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-97bl SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-97bl-b9tzq job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:04 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:05 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:07 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:07 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 101ms (101ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:07 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:07 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-97bl-b9tzq.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:20 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-97bl Completed Job completed job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:33 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-backup-lpbx-slbtj to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-smm7 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:33 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:33 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-lpbx SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-lpbx-slbtj job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:34 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 148ms (148ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:34 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:34 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:35 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:35 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 125ms (125ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:35 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:15:35 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-lpbx-slbtj.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:17:28 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-lpbx Completed Job completed job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:17:49 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:17:50 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7 job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:17:50 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:17:50 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-darling-moose/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:17:51 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-7e3a8ae8-0ae6-4260-bd78-8a106d4fcd28 pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:17:52 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:17:57 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-7e3a8ae8-0ae6-4260-bd78-8a106d4fcd28" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 125ms (125ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:03 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:04 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 158ms (158ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:04 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:04 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:04 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:05 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 149ms (149ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:05 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:05 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-tsvj7.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:20 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:20 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:22 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:22 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 110ms (110ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:22 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:23 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:24 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:24 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 122ms (122ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:24 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:24 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:48 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulDelete delete Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:49 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-bwv5 SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-bwv5-0 in StatefulSet major-upgrade-logical-replica-instance1-bwv5 successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:50 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:50 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:50 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-7ff76cfd86 SuccessfulDelete Deleted pod: major-upgrade-logical-replica-pgbouncer-7ff76cfd86-5snws replicaset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:50 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled down replica set major-upgrade-logical-replica-pgbouncer-7ff76cfd86 from 1 to 0 deployment-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:54 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-q8vs SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-q8vs-0 in StatefulSet major-upgrade-logical-replica-instance1-q8vs successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:56 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Unhealthy Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:59 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:59 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:18:59 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulDelete delete Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:03 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4 Scheduled Successfully assigned kuttl-test-darling-moose/17-to-18-pgdata-qw4m4 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-smm7 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:03 +0000 UTC Normal Job.batch 17-to-18-pgdata SuccessfulCreate Created pod: 17-to-18-pgdata-qw4m4 job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:11 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-0be6693b-50f1-49e6-8e8d-87eb53e640a3" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:13 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:13 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 145ms (145ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:13 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:13 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:14 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:47 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 33.492s (33.492s including waiting). Image size: 952996708 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:47 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4.spec.containers{database} Created Created container: database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:19:47 +0000 UTC Normal Pod 17-to-18-pgdata-qw4m4.spec.containers{database} Started Started container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:04 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-bwv5-zvqd2 Scheduled Successfully assigned kuttl-test-darling-moose/17-to-18-major-upgrade-logical-replica-instance1-bwv5-zvqd2 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:04 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-bwv5 SuccessfulCreate Created pod: 17-to-18-major-upgrade-logical-replica-instance1-bwv5-zvqd2 job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:04 +0000 UTC Normal Job.batch 17-to-18-pgdata Completed Job completed job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:09 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-bwv5-zvqd2 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-f903415a-4318-44f3-a1f4-70ecfa2e11c9" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:15 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-bwv5-zvqd2.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:47 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-bwv5-zvqd2.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 31.952s (31.952s including waiting). Image size: 952996708 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:47 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-bwv5-zvqd2.spec.containers{database} Created Created container: database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:48 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-bwv5-zvqd2.spec.containers{database} Started Started container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:20:58 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-bwv5 Completed Job completed job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-instance1-q8vs-0 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-smm7 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:00 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-jx94 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:00 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-796b69ddcd SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549 replicaset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:00 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-796b69ddcd from 0 to 1 deployment-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:00 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:01 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:01 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 176ms (176ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:01 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:01 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:02 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:02 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 132ms (132ms including waiting). Image size: 84619748 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:02 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:02 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:03 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:03 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 122ms (122ms including waiting). Image size: 84619748 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:03 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:03 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-0be6693b-50f1-49e6-8e8d-87eb53e640a3" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:05 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-bb6f52a0-003c-4992-aa9b-1d1ad495e6d8" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:06 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:06 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 135ms (135ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:06 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:06 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 128ms (128ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 111ms (111ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:07 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:08 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:08 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 146ms (146ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:08 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:08 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:08 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 124ms (124ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:08 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:08 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 144ms (144ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 133ms (133ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 148ms (148ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 125ms (125ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{database} Started Started container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:10 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 133ms (133ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:11 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-q8vs-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:16 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-instance1-bwv5-0 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-jx94 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:17 +0000 UTC Warning PerconaPGCluster.pgv2.percona.com major-upgrade-logical-replica LogicalReplicaInvalidated Logical replica "analytics" must be seeded again: the cluster was upgraded from PostgreSQL 17 to 18 after it was seeded perconapgcluster-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:19 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-backup-5ll2-xpjdb to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:19 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-5ll2 SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-5ll2-xpjdb job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 151ms (151ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:21 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:21 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 197ms (197ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:21 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:21 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-xpjdb.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:22 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-f903415a-4318-44f3-a1f4-70ecfa2e11c9" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:22 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-04 00:21:22.082 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-09-04 00:21:22.283 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? : command terminated with exit code 50 postgrescluster-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 195ms (195ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 182ms (182ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 145ms (145ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 120ms (120ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 132ms (132ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:27 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:28 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{database} Started Started container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:28 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:28 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 139ms (139ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:28 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:28 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:21:28 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-bwv5-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:33 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-backup-5ll2-l2nfc to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:33 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:33 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 132ms (132ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:33 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:33 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:33 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-5ll2 SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-5ll2-l2nfc job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:34 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:34 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 135ms (135ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:34 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:34 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-5ll2-l2nfc.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:22:50 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-5ll2 Completed Job completed job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:07 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-backup-c9hx-b7j4b to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:07 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:07 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-c9hx SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-c9hx-b7j4b job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:08 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 181ms (181ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:08 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:08 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:09 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:09 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 129ms (129ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:09 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:09 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-c9hx-b7j4b.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:21 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-c9hx Completed Job completed job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:51 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:51 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:51 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:51 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-darling-moose/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:52 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-0781cc0d-c593-474e-b26b-3ef1ae37588b pd.csi.storage.gke.io_gke-ca101010d14c4028a112-5cf6-2ab0-vm_ec6e0f6f-9946-4566-8486-0ae4fbc3d573 logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:53 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:23:59 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-0781cc0d-c593-474e-b26b-3ef1ae37588b" attachdetach-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-ee83a33b2" in 138ms (138ms including waiting). Image size: 87060666 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 154ms (154ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 100ms (100ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:02 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-dmpgb.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:15 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:16 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Scheduled Successfully assigned kuttl-test-darling-moose/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1812-ee83a33b-default-pool-ddf5ca2b-qkz6 default-scheduler logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:16 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 113ms (113ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 120ms (120ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:39 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:45 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:45 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-796b69ddcd-tk549.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:45 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | 2026-09-04 00:24:45 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 00:25:04 | major-upgrade-logical-replica | Deleting namespace "kuttl-test-darling-moose" === NAME kuttl harness.go:403: run tests finished harness.go:520: cleaning up harness.go:577: removing temp folder: "" --- PASS: kuttl (729.64s) --- PASS: kuttl/harness (0.00s) --- PASS: kuttl/harness/major-upgrade-logical-replica (728.91s) PASS