=== RUN kuttl harness.go:460: starting setup harness.go:258: running tests using configured kubeconfig. harness.go:281: Successful connection to cluster at: https://35.253.99.10 harness.go:366: running tests harness.go:77: going to run test suite with timeout of 600 seconds for each step harness.go:378: testsuite: e2e-tests/tests has 59 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-logical-replica === PAUSE kuttl/harness/major-upgrade-logical-replica === CONT kuttl/harness/major-upgrade-logical-replica logger.go:42: 22:30:59 | major-upgrade-logical-replica | Ignoring "conf": does not begin with a number followed by a dash. logger.go:42: 22:30:59 | major-upgrade-logical-replica | Creating namespace "kuttl-test-arriving-monster" logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | + source ../../functions logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ realpath ../../.. logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++ BUSYBOX_VER=1.36 logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++++ pwd logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++ test_name=major-upgrade-logical-replica logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/vars.sh logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export GIT_BRANCH=PR-1784 logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ GIT_BRANCH=PR-1784 logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export VERSION=PR-1784-d6eb16ede logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ VERSION=PR-1784-d6eb16ede logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -z '' ]] logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++++ detect_k8s_platform logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++++ local detected_platform logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++++ local xtrace_enabled=false logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++++ [[ -o xtrace ]] logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++++ xtrace_enabled=true logger.go:42: 22:30:59 | major-upgrade-logical-replica/0-deploy-operator | ++++ set +o xtrace logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ PLATFORM=gke logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export PLATFORM logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_VER=18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_DISTRIBUTION= logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_DISTRIBUTION= logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ '' == community ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export CPGO_VERSION=5.8.7 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ CPGO_VERSION=5.8.7 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export VAULT_VER=0.32.0 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ VAULT_VER=0.32.0 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ printenv logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gdate logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ which date logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gsed logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | ++++ which sed logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | + init_temp_dir logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | + deploy_operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | + local cw_prefix= logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | + destroy_operator logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 22:31:01 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | Error from server (NotFound): deployments.apps "percona-postgresql-operator" not found logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | + true logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | + create_namespace pg-operator logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | + local namespace=pg-operator logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | + [[ gke == openshift ]] logger.go:42: 22:31:02 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 22:31:03 | major-upgrade-logical-replica/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 22:31:03 | major-upgrade-logical-replica/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 22:31:04 | major-upgrade-logical-replica/0-deploy-operator | namespace/pg-operator created logger.go:42: 22:31:04 | major-upgrade-logical-replica/0-deploy-operator | + cw_prefix=cw- logger.go:42: 22:31:04 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy/crd.yaml logger.go:42: 22:31:05 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 22:31:05 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 22:31:07 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 22:31:07 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 22:31:08 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 22:31:08 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.upstream.pgv2.percona.com serverside-applied logger.go:42: 22:31:08 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.upstream.pgv2.percona.com serverside-applied logger.go:42: 22:31:10 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 22:31:10 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy/cw-rbac.yaml logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | + local disable_telemetry=true logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | + '[' major-upgrade-logical-replica == telemetry-transfer ']' logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy/cw-operator.yaml logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 22:31:11 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 22:31:12 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 22:31:12 | major-upgrade-logical-replica/0-deploy-operator | + deploy_client logger.go:42: 22:31:12 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n kuttl-test-arriving-monster apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf/client.yaml logger.go:42: 22:31:13 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 22:31:13 | major-upgrade-logical-replica/0-deploy-operator | + deploy_s3_secrets logger.go:42: 22:31:13 | major-upgrade-logical-replica/0-deploy-operator | + set +o xtrace logger.go:42: 22:31:14 | major-upgrade-logical-replica/0-deploy-operator | secret/major-upgrade-logical-replica-pgbackrest-secrets created logger.go:42: 22:31:15 | major-upgrade-logical-replica/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 22:31:15 | major-upgrade-logical-replica/0-deploy-operator | secret/do-spaces-secret created logger.go:42: 22:31:16 | major-upgrade-logical-replica/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 22:31:16 | major-upgrade-logical-replica/0-deploy-operator | secret/azure-secret created logger.go:42: 22:31:18 | major-upgrade-logical-replica/0-deploy-operator | secret/minio-secret created logger.go:42: 22:31:18 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 22:31:18 | major-upgrade-logical-replica/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 22:31:19 | major-upgrade-logical-replica/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 22:31:19 | major-upgrade-logical-replica/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 22:31:19 | major-upgrade-logical-replica/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 22:31:19 | major-upgrade-logical-replica/0-deploy-operator | ASSERT PASS logger.go:42: 22:31:19 | major-upgrade-logical-replica/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | starting test step 1-create-cluster logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions postgres_image=$(get_container_image "postgres" 17) pgbouncer_image=$(get_container_image "pgbouncer" 17) pgbackrest_image=$(get_container_image "pgbackrest" 17) get_cr \ | yq eval " .spec.postgresVersion = 17 | .spec.image = \"${postgres_image}\" | .spec.proxy.pgBouncer.image = \"${pgbouncer_image}\" | .spec.backups.pgbackrest.image = \"${pgbackrest_image}\" | .spec.patroni.removeDataDirectoryOnDivergedTimelines = true" \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | + source ../../functions logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ realpath ../../.. logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++ BUSYBOX_VER=1.36 logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++++ pwd logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++ test_name=major-upgrade-logical-replica logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/vars.sh logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export GIT_BRANCH=PR-1784 logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ GIT_BRANCH=PR-1784 logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export VERSION=PR-1784-d6eb16ede logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ VERSION=PR-1784-d6eb16ede logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -z '' ]] logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++++ detect_k8s_platform logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++++ local detected_platform logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++++ local xtrace_enabled=false logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++++ [[ -o xtrace ]] logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++++ xtrace_enabled=true logger.go:42: 22:31:19 | major-upgrade-logical-replica/1-create-cluster | ++++ set +o xtrace logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ PLATFORM=gke logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export PLATFORM logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_VER=18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ PG_VER=18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_DISTRIBUTION= logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ PG_DISTRIBUTION= logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ '' == community ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export CPGO_VERSION=5.8.7 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ CPGO_VERSION=5.8.7 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export VAULT_VER=0.32.0 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ VAULT_VER=0.32.0 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ printenv logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ which gdate logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ which date logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ which gsed logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++++ which sed logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image postgres 17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local component=postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1784-d6eb16ede =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbouncer ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbackrest ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == upgrade ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == postgis ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + postgres_image=docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbouncer 17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbouncer logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1784-d6eb16ede =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbouncer == pgbouncer ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + pgbouncer_image=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbackrest 17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbackrest logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/percona/* ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1784-d6eb16ede logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1784-d6eb16ede =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbouncer ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbackrest ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + pgbackrest_image=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + get_cr logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + local cr_name=major-upgrade-logical-replica logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + local repo_path= logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + local source_path= logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + local platform=gke logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + yq eval $'\n .spec.postgresVersion = 17 |\n .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" |\n .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" |\n .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" |\n .spec.patroni.removeDataDirectoryOnDivergedTimelines = true' logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + local cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + local platform_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + local spaces_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-spaces.yaml logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == doks ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + crs=('/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy/cr.yaml') logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + kubectl -n kuttl-test-arriving-monster apply -f - logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + local crs logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + crs+=($cr_file) logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + yq eval-all $'\n\t\tselect(fileIndex == 0) * (select(fileIndex == 1) // {}) |\n\t\t.metadata.name = "major-upgrade-logical-replica" |\n\t\t.metadata.labels = {"e2e":"major-upgrade-logical-replica"} |\n\t\t.spec.postgresVersion = 18 |\n\t\t.spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.users += [{"name":"major-upgrade-logical-replica","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" |\n\t\t.spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" |\n\t\t.spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" |\n\t\t.spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" |\n\t\t.spec.logcollector.image = "docker.io/perconalab/fluentbit:main-logcollector" |\n\t\t.spec.pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest" |\n\t\t.spec.pmm.secret = "major-upgrade-logical-replica-pmm-secret" |\n\t\t.spec.pmm.customClusterName = "major-upgrade-logical-replica-pmm-custom-name" |\n\t\t.spec.pmm.postgresParams = "--environment=dev-postgres"\n\t' /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy/cr.yaml /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||pg-operator-testing|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == openshift ]] logger.go:42: 22:31:21 | major-upgrade-logical-replica/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 22:31:22 | major-upgrade-logical-replica/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica created logger.go:42: 22:32:19 | major-upgrade-logical-replica/1-create-cluster | test step completed 1-create-cluster logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | starting test step 2-write-data logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:${PASS}@${HOST}" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:${PASS}@${HOST}"] logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | + source ../../functions logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ realpath ../../.. logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++ BUSYBOX_VER=1.36 logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++++ pwd logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++ test_name=major-upgrade-logical-replica logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/vars.sh logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export GIT_BRANCH=PR-1784 logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ GIT_BRANCH=PR-1784 logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export VERSION=PR-1784-d6eb16ede logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ VERSION=PR-1784-d6eb16ede logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | +++ [[ -z '' ]] logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++++ detect_k8s_platform logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++++ local detected_platform logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++++ local xtrace_enabled=false logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++++ [[ -o xtrace ]] logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++++ xtrace_enabled=true logger.go:42: 22:32:19 | major-upgrade-logical-replica/2-write-data | ++++ set +o xtrace logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ PLATFORM=gke logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export PLATFORM logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ -n '' ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export PG_VER=18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ PG_VER=18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export PG_DISTRIBUTION= logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ PG_DISTRIBUTION= logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ '' == community ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_VER=14 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export CPGO_VERSION=5.8.7 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ CPGO_VERSION=5.8.7 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export VAULT_VER=0.32.0 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ VAULT_VER=0.32.0 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ printenv logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ which gdate logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ which date logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ date=/usr/sbin/date logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ which gsed logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++++ which sed logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-arriving-monster get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | + PASS=yPaDArJx17qUXfHI4vncpSAE logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:32:21 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-arriving-monster get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 22:32:22 | major-upgrade-logical-replica/2-write-data | + HOST=major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:32:22 | major-upgrade-logical-replica/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:32:22 | major-upgrade-logical-replica/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 22:32:22 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:32:22 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 22:32:22 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 22:32:22 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-arriving-monster get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 22:32:22 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-arriving-monster exec pg-client-d7c7bb45b-8vfvv -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc'\''' logger.go:42: 22:32:23 | major-upgrade-logical-replica/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:32:23 | major-upgrade-logical-replica/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 22:32:23 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:32:23 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 22:32:23 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 22:32:23 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-arriving-monster get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 22:32:24 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-arriving-monster exec pg-client-d7c7bb45b-8vfvv -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc'\''' logger.go:42: 22:32:25 | major-upgrade-logical-replica/2-write-data | test step completed 2-write-data logger.go:42: 22:32:25 | major-upgrade-logical-replica/3-create-seed-backup | starting test step 3-create-seed-backup logger.go:42: 22:32:25 | major-upgrade-logical-replica/3-create-seed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # The pgbackrest bootstrap Job restores the latest full backup, and the # one taken at cluster creation predates the rows written above. Take a # new one so the seeded data is actually in the backup being restored. cat </dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:06 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:08 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:09 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:13 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:15 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:17 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:19 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:20 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:22 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:24 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:25 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:25 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:28 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:28 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:29 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:31 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:32 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:33 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:35 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:35 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:36 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:39 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:40 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:43 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:44 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:46 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:46 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:47 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:49 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:50 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:51 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:53 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:54 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:54 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:36:57 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:36:57 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:36:58 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:37:00 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:01 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:02 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:37:04 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:04 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:05 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:37:07 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:08 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:09 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:37:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:12 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:12 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:37:15 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:15 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:16 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-kx984" is waiting to start: PodInitializing logger.go:42: 22:37:18 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:19 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:20 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:22 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:23 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:26 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:26 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:27 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:29 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:30 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:31 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:37:33 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:33 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:34 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:37:36 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:37 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:38 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:37:40 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:41 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:37:44 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:44 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:45 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:37:47 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:48 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:49 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:37:51 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:52 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:37:55 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:55 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:37:56 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:37:58 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:37:59 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:00 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:02 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:03 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:04 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:06 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:07 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:10 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:10 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:11 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:13 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:14 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:15 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:17 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:18 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:21 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:21 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:22 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:24 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:25 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:26 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:28 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:29 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:30 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:32 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:32 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:33 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:35 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:36 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:37 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:39 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:40 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:43 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:43 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:44 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:46 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:47 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:48 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:50 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:50 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:51 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:54 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:54 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:55 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:38:57 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:38:58 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:38:59 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:39:01 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | real 0m1.693s logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | user 0m0.093s logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | sys 0m0.109s logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | real 0m9.508s logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | user 0m0.248s logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | sys 0m0.485s logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 22:39:02 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 22:39:03 | major-upgrade-logical-replica/5-upgrade | test step completed 5-upgrade logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | starting test step 6-verify-replica-invalidated logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-logical-replica primary name) kubectl -n "${NAMESPACE}" exec "${primary}" -- /pgdata/delete_old_cluster.sh # pg_upgrade only rewrites the data directories of the instance sets, so the # replica still holds a PostgreSQL 17 one. It has to be reported as broken # rather than handed the new binary and left to crash. retry 60 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") reason=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].reason}") echo "logical replica analytics: ${state}/${reason}" [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]] ' message=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].message}') echo "message: ${message}" echo "${message}" | grep -q "remove it from spec.logicalReplicas" # The StatefulSet is stopped, so nothing is left crash-looping against a data # directory the new binary cannot open. retry 30 5 bash -c ' replicas=$(kubectl -n "'"${NAMESPACE}"'" get statefulset \ major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}") echo "logical replica statefulset replicas: ${replicas}" [[ ${replicas} == "0" ]] ' # pg_upgrade carries the logical slots over to the new cluster. Nothing will # ever read from them again, so they must not be left pinning WAL. PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) no_slots_left() { local got got=$(run_psql \ "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ "postgres://postgres:${PASS}@${HOST}" | xargs) echo "logical slots left on the primary: ${got}" [[ ${got} == "0" ]] } retry 30 5 no_slots_left] logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | + source ../../functions logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ realpath ../../.. logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ BUSYBOX_VER=1.36 logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ pwd logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ test_name=major-upgrade-logical-replica logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/vars.sh logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export GIT_BRANCH=PR-1784 logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ GIT_BRANCH=PR-1784 logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VERSION=PR-1784-d6eb16ede logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VERSION=PR-1784-d6eb16ede logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ SKIP_TEST_WARNINGS=true logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -z '' ]] logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ detect_k8s_platform logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local detected_platform logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local xtrace_enabled=false logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ [[ -o xtrace ]] logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ xtrace_enabled=true logger.go:42: 22:39:03 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ set +o xtrace logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PLATFORM=gke logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PLATFORM logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ ! -n '' ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -n '' ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_VER=18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_VER=18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_DISTRIBUTION= logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_DISTRIBUTION= logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ '' == community ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export BUCKET=pg-operator-testing logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ BUCKET=pg-operator-testing logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_TAG=1.4.0 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_TAG=1.4.0 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_VER=14 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_VER=14 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export CPGO_VERSION=5.8.7 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ CPGO_VERSION=5.8.7 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export MINIO_VER=5.4.0 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ MINIO_VER=5.4.0 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VAULT_VER=0.32.0 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VAULT_VER=0.32.0 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME=docker.io logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME=docker.io logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ printenv logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ grep -E '^IMAGE' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ awk -F= '{print $1}' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gdate logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which date logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ date=/usr/sbin/date logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gsed logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which sed logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ sed=/usr/sbin/sed logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_pod_by_role major-upgrade-logical-replica primary name logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local cluster=major-upgrade-logical-replica logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local role=primary logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local parameter=name logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ case ${parameter} in logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'jsonpath={.items[].metadata.name}' logger.go:42: 22:39:05 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl get pods --namespace kuttl-test-arriving-monster --selector=postgres-operator.crunchydata.com/role=primary,postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica -o 'jsonpath={.items[].metadata.name}' logger.go:42: 22:39:06 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ echo major-upgrade-logical-replica-instance1-tt8m-0 logger.go:42: 22:39:06 | major-upgrade-logical-replica/6-verify-replica-invalidated | + primary=major-upgrade-logical-replica-instance1-tt8m-0 logger.go:42: 22:39:06 | major-upgrade-logical-replica/6-verify-replica-invalidated | + kubectl -n kuttl-test-arriving-monster exec major-upgrade-logical-replica-instance1-tt8m-0 -- /pgdata/delete_old_cluster.sh logger.go:42: 22:39:07 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 60 10 bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 22:39:07 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=60 logger.go:42: 22:39:07 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=10 logger.go:42: 22:39:07 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 22:39:07 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 22:39:07 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 22:39:08 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica analytics: broken/SourceUpgraded logger.go:42: 22:39:08 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-arriving-monster get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].message}' logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + message='this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + grep -q 'remove it from spec.logicalReplicas' logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 bash -c $'\n replicas=$(kubectl -n "kuttl-test-arriving-monster" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 22:39:09 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n replicas=$(kubectl -n "kuttl-test-arriving-monster" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 22:39:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica statefulset replicas: 0 logger.go:42: 22:39:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:39:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:39:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-arriving-monster get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 22:39:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | + PASS=yPaDArJx17qUXfHI4vncpSAE logger.go:42: 22:39:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:39:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:39:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-arriving-monster get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | + HOST=major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 no_slots_left logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | + no_slots_left logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local got logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ run_psql 'SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ xargs logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'command=SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local uri=postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local password= logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ get_client_pod logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl -n kuttl-test-arriving-monster get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 22:39:11 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-arriving-monster exec pg-client-d7c7bb45b-8vfvv -- bash -c 'printf '\''SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc' logger.go:42: 22:39:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | + got=0 logger.go:42: 22:39:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'logical slots left on the primary: 0' logger.go:42: 22:39:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical slots left on the primary: 0 logger.go:42: 22:39:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | + [[ 0 == 0 ]] logger.go:42: 22:39:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | test step completed 6-verify-replica-invalidated logger.go:42: 22:39:12 | major-upgrade-logical-replica/7-create-reseed-backup | starting test step 7-create-reseed-backup logger.go:42: 22:39:12 | major-upgrade-logical-replica/7-create-reseed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # pg_upgrade left the repository holding only pre-upgrade backups, so the # bootstrap Job of the reseed below would restore a PostgreSQL 17 data # directory. Take a full backup of the upgraded cluster first. cat </dev/null 2>&1 && return 1 kubectl -n "${NAMESPACE}" get pvc major-upgrade-logical-replica-lr-analytics-pgdata >/dev/null 2>&1 && return 1 return 0 } retry 60 5 status_gone retry 60 5 objects_gone kubectl -n "${NAMESPACE}" patch pg major-upgrade-logical-replica --type=merge --patch '{ "spec": { "logicalReplicas": [ { "name": "analytics", "databases": ["myapp"], "bootstrapMethod": "pgbackrest", "dataVolumeClaimSpec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}} } } ] } }'] logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/vars.sh logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1784 logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1784 logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1784-d6eb16ede logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1784-d6eb16ede logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 22:40:03 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 22:40:05 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-arriving-monster patch pg major-upgrade-logical-replica --type=json --patch '[{"op": "remove", "path": "/spec/logicalReplicas"}]' logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 status_gone logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-arriving-monster get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + got='[{"invalidatedAt":"2026-09-03T22:38:10Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T22:35:31Z","state":"broken"}]' logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\''[{"invalidatedAt":"2026-09-03T22:38:10Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T22:35:31Z","state":"broken"}]'\''' logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '[{"invalidatedAt":"2026-09-03T22:38:10Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T22:35:31Z","state":"broken"}]' logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z [{"invalidatedAt":"2026-09-03T22:38:10Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T22:35:31Z","state":"broken"}] ]] logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ [{"invalidatedAt":"2026-09-03T22:38:10Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T22:35:31Z","state":"broken"}] == \[\] ]] logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 60 ]] logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 22:40:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 5 logger.go:42: 22:40:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 22:40:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 22:40:11 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-arriving-monster get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + got= logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\'''\''' logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '' logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z '' ]] logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 objects_gone logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + objects_gone logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-arriving-monster get statefulset major-upgrade-logical-replica-lr-analytics logger.go:42: 22:40:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-arriving-monster get pvc major-upgrade-logical-replica-lr-analytics-pgdata logger.go:42: 22:40:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + return 0 logger.go:42: 22:40:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-arriving-monster patch pg major-upgrade-logical-replica --type=merge --patch $'{\n "spec": {\n "logicalReplicas": [\n {\n "name": "analytics",\n "databases": ["myapp"],\n "bootstrapMethod": "pgbackrest",\n "dataVolumeClaimSpec": {\n "accessModes": ["ReadWriteOnce"],\n "resources": {"requests": {"storage": "1Gi"}}\n }\n }\n ]\n }\n}' logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | running command: [sh -c set -o errexit set -o xtrace source ../../functions retry 90 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") echo "logical replica analytics state: ${state}" [[ ${state} == "ready" ]] ' # Seeded from the upgraded cluster, so it is stamped with the new version. seeded=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}') echo "seeded at PostgreSQL: ${seeded}" [[ ${seeded} == "18" ]] kubectl -n "${NAMESPACE}" wait --for=condition=Ready pod \ -l "pgv2.percona.com/logical-replica=analytics" --timeout=300s PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) REPLICA="major-upgrade-logical-replica-lr-analytics.${NAMESPACE}.svc" # Replication actually streams again on the upgraded cluster. run_psql \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' \ "postgres://postgres:${PASS}@${HOST}" row_streamed() { local got got=$(run_psql \ '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' \ "postgres://postgres:${PASS}@${REPLICA}" | xargs) echo "streamed rows on the replica: ${got}" [[ ${got} == "1" ]] } retry 30 5 row_streamed] logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/vars.sh logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1784 logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1784 logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1784-d6eb16ede logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1784-d6eb16ede logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 22:40:14 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 90 10 bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=90 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=10 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 90 ]] logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 22:40:16 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 22:40:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 22:40:27 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 22:40:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 2 -ge 90 ]] logger.go:42: 22:40:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 22:40:27 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 22:40:37 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 22:40:37 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 22:40:37 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 3 -ge 90 ]] logger.go:42: 22:40:37 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 22:40:37 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 22:40:47 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 22:40:48 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 22:40:48 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 4 -ge 90 ]] logger.go:42: 22:40:48 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 22:40:48 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 22:40:58 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 22:40:58 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 22:40:58 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 5 -ge 90 ]] logger.go:42: 22:40:58 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 22:40:58 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 22:41:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-arriving-monster" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 22:41:09 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: ready logger.go:42: 22:41:09 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-arriving-monster get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}' logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + seeded=18 logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'seeded at PostgreSQL: 18' logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | seeded at PostgreSQL: 18 logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 18 == 18 ]] logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-arriving-monster wait --for=condition=Ready pod -l pgv2.percona.com/logical-replica=analytics --timeout=300s logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | pod/major-upgrade-logical-replica-lr-analytics-0 condition met logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:41:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-arriving-monster get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + PASS=yPaDArJx17qUXfHI4vncpSAE logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-arriving-monster get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + HOST=major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + REPLICA=major-upgrade-logical-replica-lr-analytics.kuttl-test-arriving-monster.svc logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + local uri=postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + local password= logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_client_pod logger.go:42: 22:41:11 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-arriving-monster get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 22:41:12 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-arriving-monster exec pg-client-d7c7bb45b-8vfvv -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-primary.kuttl-test-arriving-monster.svc' logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 30 5 row_streamed logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=30 logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + row_streamed logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ run_psql '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-lr-analytics.kuttl-test-arriving-monster.svc logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local 'command=\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local uri=postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-lr-analytics.kuttl-test-arriving-monster.svc logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local password= logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ xargs logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ get_client_pod logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ kubectl -n kuttl-test-arriving-monster get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 22:41:13 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-arriving-monster exec pg-client-d7c7bb45b-8vfvv -- bash -c 'printf '\''\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:yPaDArJx17qUXfHI4vncpSAE@major-upgrade-logical-replica-lr-analytics.kuttl-test-arriving-monster.svc' logger.go:42: 22:41:14 | major-upgrade-logical-replica/8-reseed-logical-replica | + got=1 logger.go:42: 22:41:14 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'streamed rows on the replica: 1' logger.go:42: 22:41:14 | major-upgrade-logical-replica/8-reseed-logical-replica | streamed rows on the replica: 1 logger.go:42: 22:41:14 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 == 1 ]] logger.go:42: 22:41:14 | major-upgrade-logical-replica/8-reseed-logical-replica | test step completed 8-reseed-logical-replica logger.go:42: 22:41:14 | major-upgrade-logical-replica/99-remove-cluster-gracefully | starting test step 99-remove-cluster-gracefully logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | running command: [sh -c set -o errexit set -o xtrace source ../../functions remove_all_finalizers check_operator_panic destroy_operator] logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + source ../../functions logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ realpath ../../.. logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ BUSYBOX_VER=1.36 logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ pwd logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ test_name=major-upgrade-logical-replica logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/vars.sh logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784 logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/deploy logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/conf logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export GIT_BRANCH=PR-1784 logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ GIT_BRANCH=PR-1784 logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VERSION=PR-1784-d6eb16ede logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VERSION=PR-1784-d6eb16ede logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ SKIP_TEST_WARNINGS=true logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -z '' ]] logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ detect_k8s_platform logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local detected_platform logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local xtrace_enabled=false logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ [[ -o xtrace ]] logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ xtrace_enabled=true logger.go:42: 22:41:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ set +o xtrace logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PLATFORM=gke logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PLATFORM logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ ! -n '' ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -n '' ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_VER=18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_VER=18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_DISTRIBUTION= logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_DISTRIBUTION= logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ '' == community ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export BUCKET=pg-operator-testing logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ BUCKET=pg-operator-testing logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_TAG=1.4.0 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_TAG=1.4.0 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_VER=14 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_VER=14 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export CPGO_VERSION=5.8.7 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ CPGO_VERSION=5.8.7 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export MINIO_VER=5.4.0 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ MINIO_VER=5.4.0 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VAULT_VER=0.32.0 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VAULT_VER=0.32.0 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME=docker.io logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME=docker.io logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ printenv logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ grep -E '^IMAGE' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ awk -F= '{print $1}' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1784-d6eb16ede == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gdate logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which date logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ date=/usr/sbin/date logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gsed logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1784/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which sed logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ sed=/usr/sbin/sed logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + remove_all_finalizers logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + resource_types=("pg-restore" "pg-backup" "pg") logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-restore resources' logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-restore resources logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-arriving-monster get pg-restore -o json logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 22:41:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-backup resources' logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-backup resources logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-arriving-monster get pg-backup -o json logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-arriving-monster get pg-backup major-upgrade-logical-replica-backup-6msq-rwmq4 -o yaml logger.go:42: 22:41:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-arriving-monster get pg-backup major-upgrade-logical-replica-backup-p2sg-vnt4g -o yaml logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-arriving-monster get pg-backup major-upgrade-logical-replica-reseed-repo1 -o yaml logger.go:42: 22:41:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-arriving-monster get pg-backup major-upgrade-logical-replica-seed-repo1 -o yaml logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg resources' logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg resources logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-arriving-monster get pg -o json logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 22:41:26 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 22:41:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + check_operator_panic logger.go:42: 22:41:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ get_operator_pod logger.go:42: 22:41:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ kubectl get pods -n pg-operator --selector=app.kubernetes.io/name=percona-postgresql-operator -o 'jsonpath={.items[].metadata.name}' logger.go:42: 22:41:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ echo percona-postgresql-operator-55b44564f5-wctwj logger.go:42: 22:41:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local operator_pod=percona-postgresql-operator-55b44564f5-wctwj logger.go:42: 22:41:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local panic_log logger.go:42: 22:41:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl logs -n pg-operator percona-postgresql-operator-55b44564f5-wctwj -c operator logger.go:42: 22:41:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ grep -A 100 'Observed a panic' logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ true logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + panic_log= logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + '[' -n '' ']' logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + destroy_operator logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ -n pg-operator ]] logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 22:41:29 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 22:41:30 | major-upgrade-logical-replica/99-remove-cluster-gracefully | namespace "pg-operator" force deleted logger.go:42: 22:41:41 | major-upgrade-logical-replica/99-remove-cluster-gracefully | test step completed 99-remove-cluster-gracefully logger.go:42: 22:41:41 | major-upgrade-logical-replica | major-upgrade-logical-replica events from ns kuttl-test-arriving-monster: logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:13 +0000 UTC Normal Pod pg-client-d7c7bb45b-8vfvv Scheduled Successfully assigned kuttl-test-arriving-monster/pg-client-d7c7bb45b-8vfvv to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-77ps default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:13 +0000 UTC Normal ReplicaSet.apps pg-client-d7c7bb45b SuccessfulCreate Created pod: pg-client-d7c7bb45b-8vfvv replicaset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:13 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-d7c7bb45b from 0 to 1 deployment-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:14 +0000 UTC Normal Pod pg-client-d7c7bb45b-8vfvv.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:14 +0000 UTC Normal Pod pg-client-d7c7bb45b-8vfvv.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 151ms (152ms including waiting). Image size: 445181537 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:14 +0000 UTC Normal Pod pg-client-d7c7bb45b-8vfvv.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:14 +0000 UTC Normal Pod pg-client-d7c7bb45b-8vfvv.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:23 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-ngkx-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:23 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-ngkx-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:23 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-ngkx-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-arriving-monster/major-upgrade-logical-replica-instance1-ngkx-pgdata" pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:23 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-ngkx SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-ngkx-0 in StatefulSet major-upgrade-logical-replica-instance1-ngkx successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:23 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-tt8m-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:24 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-tt8m-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:24 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-tt8m-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-arriving-monster/major-upgrade-logical-replica-instance1-tt8m-pgdata" pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:24 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-tt8m SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-tt8m-0 in StatefulSet major-upgrade-logical-replica-instance1-tt8m successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:24 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulCreate create Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:24 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica RepoHostCreated created pgBackRest repository host StatefulSet/major-upgrade-logical-replica-repo-host postgrescluster-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-instance1-ngkx-0 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-ngkx-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-6aff1d9c-c740-4b83-8695-9e9d2cc24500 pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-tt8m-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-afa7c9de-e122-4493-883b-5d9acc2213c0 pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-qk4z default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-7cc76dcc5d SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d replicaset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-7cc76dcc5d from 0 to 1 deployment-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Warning Pod major-upgrade-logical-replica-repo-host-0 FailedScheduling 0/3 nodes are available: persistentvolumeclaim "major-upgrade-logical-replica-repo1" not found. not found default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 WaitForPodScheduled waiting for pod major-upgrade-logical-replica-repo-host-0 to be scheduled persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:25 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-arriving-monster/major-upgrade-logical-replica-repo1" pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-instance1-tt8m-0 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-77ps default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:26 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:26 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 206ms (206ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:26 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:27 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:27 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-f1a6ee69-0dbb-4008-8122-ec2fe17a3c01 pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:28 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-qk4z default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:31 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 3.295s (3.295s including waiting). Image size: 84606212 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:31 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:31 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:31 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:31 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 166ms (166ms including waiting). Image size: 84606212 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:31 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:31 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-6aff1d9c-c740-4b83-8695-9e9d2cc24500" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-afa7c9de-e122-4493-883b-5d9acc2213c0" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 162ms (162ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 113ms (113ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:35 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:35 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:36 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-f1a6ee69-0dbb-4008-8122-ec2fe17a3c01" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:38 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:43 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 5.665s (5.665s including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:43 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:44 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:47 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:47 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 125ms (125ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:47 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:47 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 125ms (125ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 110ms (110ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:48 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:51 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 15.795s (15.795s including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:51 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:51 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:53 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 17.715s (17.715s including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:53 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:53 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 141ms (141ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:31:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 115ms (115ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 183ms (183ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 118ms (118ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{database} Started Started container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 128ms (128ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{logs} Pulling Pulling image "docker.io/perconalab/fluentbit:main-logcollector" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 151ms (151ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 165ms (165ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Started Started container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 137ms (137ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:07 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:08 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-backup-p2sg-8pb2p to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:08 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:08 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 159ms (159ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:08 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:08 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-p2sg SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-p2sg-8pb2p job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:09 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:11 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:12 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 173ms (173ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:12 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:12 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-p2sg-8pb2p.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:26 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-p2sg Completed Job completed job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:41 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-backup-rfvp-9l66j to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:41 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-rfvp SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-rfvp-9l66j job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:42 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:42 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 158ms (158ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:42 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:42 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:44 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:44 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 139ms (139ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:44 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:32:44 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-rfvp-9l66j.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:34:35 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-rfvp Completed Job completed job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:02 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:03 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:03 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:03 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-arriving-monster/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:04 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-fcc8fe9d-ce29-485e-8ffe-5cedbc4becb8 pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:05 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:11 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-fcc8fe9d-ce29-485e-8ffe-5cedbc4becb8" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:13 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:13 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 187ms (187ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:13 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:13 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:14 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:14 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 178ms (178ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:14 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:14 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:15 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:15 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 132ms (132ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:15 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:15 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-5vnmm.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:30 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-77ps default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:31 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:41 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-fcc8fe9d-ce29-485e-8ffe-5cedbc4becb8" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:42 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:43 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 124ms (124ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:43 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:43 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:43 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:44 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 112ms (112ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:44 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:35:44 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:09 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:09 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulDelete delete Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:10 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-ngkx SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-ngkx-0 in StatefulSet major-upgrade-logical-replica-instance1-ngkx successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:11 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled down replica set major-upgrade-logical-replica-pgbouncer-7cc76dcc5d from 1 to 0 deployment-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:12 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:12 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:12 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-7cc76dcc5d SuccessfulDelete Deleted pod: major-upgrade-logical-replica-pgbouncer-7cc76dcc5d-jgz6d replicaset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:16 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-tt8m SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-tt8m-0 in StatefulSet major-upgrade-logical-replica-instance1-tt8m successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:19 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-logical-replica-set-instance1 NoPods No matching pods found controllermanager logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:22 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:22 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:22 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulDelete delete Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:24 +0000 UTC Normal Pod 17-to-18-pgdata-kx984 Scheduled Successfully assigned kuttl-test-arriving-monster/17-to-18-pgdata-kx984 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:24 +0000 UTC Warning Pod 17-to-18-pgdata-kx984 FailedAttachVolume Multi-Attach error for volume "pvc-afa7c9de-e122-4493-883b-5d9acc2213c0" Volume is already exclusively attached to one node and can't be attached to another attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:24 +0000 UTC Normal Job.batch 17-to-18-pgdata SuccessfulCreate Created pod: 17-to-18-pgdata-kx984 job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:38 +0000 UTC Normal Pod 17-to-18-pgdata-kx984 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-afa7c9de-e122-4493-883b-5d9acc2213c0" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:40 +0000 UTC Normal Pod 17-to-18-pgdata-kx984.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:40 +0000 UTC Normal Pod 17-to-18-pgdata-kx984.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 134ms (134ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:40 +0000 UTC Normal Pod 17-to-18-pgdata-kx984.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:40 +0000 UTC Normal Pod 17-to-18-pgdata-kx984.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:36:41 +0000 UTC Normal Pod 17-to-18-pgdata-kx984.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:16 +0000 UTC Normal Pod 17-to-18-pgdata-kx984.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 35.439s (35.439s including waiting). Image size: 952996708 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:16 +0000 UTC Normal Pod 17-to-18-pgdata-kx984.spec.containers{database} Created Created container: database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:16 +0000 UTC Normal Pod 17-to-18-pgdata-kx984.spec.containers{database} Started Started container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:33 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-ngkx-drr59 Scheduled Successfully assigned kuttl-test-arriving-monster/17-to-18-major-upgrade-logical-replica-instance1-ngkx-drr59 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:33 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-ngkx SuccessfulCreate Created pod: 17-to-18-major-upgrade-logical-replica-instance1-ngkx-drr59 job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:33 +0000 UTC Normal Job.batch 17-to-18-pgdata Completed Job completed job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:42 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-ngkx-drr59 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-6aff1d9c-c740-4b83-8695-9e9d2cc24500" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:45 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-ngkx-drr59.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:45 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-ngkx-drr59.spec.containers{database} Created Created container: database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:45 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-ngkx-drr59.spec.containers{database} Started Started container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:47 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-ngkx Completed Job completed job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:53 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-instance1-tt8m-0 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:53 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-77ps default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:54 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-qk4z default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:54 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:54 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-7957d85cb6 SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7 replicaset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:54 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-7957d85cb6 from 0 to 1 deployment-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:55 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 148ms (148ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:55 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:55 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 117ms (117ms including waiting). Image size: 84619748 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 155ms (155ms including waiting). Image size: 84619748 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-afa7c9de-e122-4493-883b-5d9acc2213c0" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 183ms (183ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:37:59 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-f1a6ee69-0dbb-4008-8122-ec2fe17a3c01" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 138ms (138ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 151ms (151ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 126ms (126ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:02 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:02 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 100ms (100ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:02 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:02 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 126ms (126ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{database} Started Started container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 164ms (164ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 106ms (106ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:03 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-tt8m-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 90ms (90ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 112ms (112ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:04 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:09 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-instance1-ngkx-0 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-qk4z default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:10 +0000 UTC Warning PerconaPGCluster.pgv2.percona.com major-upgrade-logical-replica LogicalReplicaInvalidated Logical replica "analytics" must be seeded again: the cluster was upgraded from PostgreSQL 17 to 18 after it was seeded perconapgcluster-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-6aff1d9c-c740-4b83-8695-9e9d2cc24500" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-backup-6msq-drd7m to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-77ps default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 116ms (116ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-6msq SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-6msq-drd7m job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 150ms (150ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:15 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:15 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 102ms (102ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:15 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:15 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6msq-drd7m.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:17 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-03 22:38:17.488 P00 ERROR: [050]: raised from remote-0 tls protocol on 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-arriving-monster.svc.cluster.local': unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-09-03 22:38:17.703 P00 ERROR: [050]: raised from remote-0 tls protocol on 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-arriving-monster.svc.cluster.local': unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? : command terminated with exit code 50 postgrescluster-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 16.765s (16.765s including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 121ms (121ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 141ms (141ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 124ms (124ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Started Started container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:43 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 167ms (167ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:43 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:43 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:38:43 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{logs} Pulling Pulling image "docker.io/perconalab/fluentbit:main-logcollector" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:27 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-6msq Completed Job completed job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:37 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-backup-hd4x-tz5q4 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-77ps default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:37 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-hd4x SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-hd4x-tz5q4 job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 132ms (132ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:38 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 112ms (112ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:39 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-hd4x-tz5q4.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:39:47 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-hd4x Completed Job completed job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:18 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:18 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:18 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:18 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-arriving-monster/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:20 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:20 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-e74b08b5-844a-47ba-8090-65d1e937e17d pd.csi.storage.gke.io_gke-91c2545e0e8e41d1ba31-ff6a-9a5e-vm_020688c8-6b6c-47a7-924c-053036c03054 logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:25 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-e74b08b5-844a-47ba-8090-65d1e937e17d" attachdetach-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:28 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:28 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1784-d6eb16ede" in 149ms (149ms including waiting). Image size: 87067853 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:28 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:28 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 160ms (160ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 132ms (132ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-jjslc.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:43 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:45 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Scheduled Successfully assigned kuttl-test-arriving-monster/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1784-d6eb16ed-default-pool-b843f6db-xzf1 default-scheduler logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:45 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 124ms (124ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 137ms (137ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:40:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:41:16 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:41:17 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-ngkx-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:41:22 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:41:22 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-7957d85cb6-xqwk7.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:41:22 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | 2026-09-03 22:41:22 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 22:41:41 | major-upgrade-logical-replica | Deleting namespace "kuttl-test-arriving-monster" === NAME kuttl harness.go:404: run tests finished harness.go:511: cleaning up harness.go:568: removing temp folder: "" --- PASS: kuttl (708.23s) --- PASS: kuttl/harness (0.00s) --- PASS: kuttl/harness/major-upgrade-logical-replica (707.46s) PASS